samples: video: capture: Add support for m2m transformation

Sometimnes, additional transforms needs to be applied on the video frame
to be able to display it correctly on the LCD.

This is to add support for such a transformation. This also adds a
90-degree image rotation using PxP m2m device on i.MX RT1170 as an
example of such usecase.

Signed-off-by: Phi Bang Nguyen <phibang.nguyen@nxp.com>

Author: ngphibang

samples/drivers/video/capture/CMakeLists.txt

@@ -6,3 +6,7 @@ project(video_capture)
 
 FILE(GLOB app_sources src/*.c)
 target_sources(app PRIVATE ${app_sources})
+
+if(CONFIG_VIDEO_TRANSFORM)
+    target_sources(app PRIVATE boards/${BOARD}/transform.c)
+endif()

samples/drivers/video/capture/Kconfig

@@ -58,6 +58,11 @@ config VIDEO_CTRL_VFLIP
 	help
 	  If set, mirror the video frame vertically
 
+config VIDEO_TRANSFORM
+	bool "Perform some m2m transforms on the video frame"
+	help
+	  If set, the video frame will go through some m2m transforms before being displayed
+
 endmenu
 
 source "Kconfig.zephyr"

samples/drivers/video/capture/boards/mimxrt1170_evk/transform.c

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2025 NXP
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/device.h>
+#include <zephyr/drivers/video.h>
+#include <zephyr/drivers/video-controls.h>
+
+#include <zephyr/logging/log.h>
+
+LOG_MODULE_REGISTER(transform, CONFIG_LOG_DEFAULT_LEVEL);
+
+static const struct device *transform_dev;
+static struct video_buffer *transformed_buffer;
+
+int video_transform_setup(struct video_format in_fmt, struct video_format *out_fmt)
+{
+	int ret;
+	struct video_control ctrl = {.id = VIDEO_CID_ROTATE, .val = 90};
+
+	transform_dev = DEVICE_DT_GET(DT_NODELABEL(pxp));
+
+	out_fmt->pixelformat = in_fmt.pixelformat;
+	out_fmt->width = in_fmt.height;
+	out_fmt->height = in_fmt.width;
+	out_fmt->type = VIDEO_BUF_TYPE_OUTPUT;
+
+	/* Set input and output formats */
+	in_fmt.type = VIDEO_BUF_TYPE_INPUT;
+	ret = video_set_format(transform_dev, &in_fmt);
+	if (ret) {
+		LOG_ERR("Unable to set input format");
+		return ret;
+	}
+
+	ret = video_set_format(transform_dev, out_fmt);
+	if (ret) {
+		LOG_ERR("Unable to set output format");
+		return ret;
+	}
+
+	/* Set controls */
+	video_set_ctrl(transform_dev, &ctrl);
+
+	/* Allocate a buffer for output queue */
+	transformed_buffer = video_buffer_aligned_alloc(in_fmt.pitch * in_fmt.height,
+						CONFIG_VIDEO_BUFFER_POOL_ALIGN, K_FOREVER);
+
+	if (transformed_buffer == NULL) {
+		LOG_ERR("Unable to alloc video buffer");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+int video_transform_process(struct video_buffer *in_buf, struct video_buffer **out_buf)
+{
+	int err;
+	struct video_buffer *transformed_buf = &(struct video_buffer){};
+	/* Store the input buffer type to recover later */
+	enum video_buf_type in_buf_type = in_buf->type;
+
+	in_buf->type = VIDEO_BUF_TYPE_INPUT;
+	err = video_enqueue(transform_dev, in_buf);
+	if (err) {
+		LOG_ERR("Unable to enqueue input buffer");
+		return err;
+	}
+
+	transformed_buffer->type = VIDEO_BUF_TYPE_OUTPUT;
+	err = video_enqueue(transform_dev, transformed_buffer);
+	if (err) {
+		LOG_ERR("Unable to enqueue output buffer");
+		return err;
+	}
+
+	/* Same start for both input and output */
+	if (video_stream_start(transform_dev, VIDEO_BUF_TYPE_INPUT)) {
+		LOG_ERR("Unable to start PxP");
+		return 0;
+	}
+
+	/* Dequeue input and output buffers */
+	err = video_dequeue(transform_dev, &in_buf, K_FOREVER);
+	if (err) {
+		LOG_ERR("Unable to dequeue PxP video buf in");
+		return err;
+	}
+
+	transformed_buf->type = VIDEO_BUF_TYPE_OUTPUT;
+	err = video_dequeue(transform_dev, &transformed_buf, K_FOREVER);
+	if (err) {
+		LOG_ERR("Unable to dequeue PxP video buf out");
+		return err;
+	}
+
+	*out_buf = transformed_buf;
+
+	/* Keep the input buffer intact */
+	in_buf->type = in_buf_type;
+
+	return 0;
+}

samples/drivers/video/capture/boards/mimxrt1170_evk_mimxrt1176_cm7.conf

@@ -1 +1,3 @@
-CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=3686800
+CONFIG_VIDEO_TRANSFORM=y
+CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=6530200
+CONFIG_VIDEO_BUFFER_POOL_NUM_MAX=3

samples/drivers/video/capture/src/main.c

@@ -26,6 +26,22 @@ LOG_MODULE_REGISTER(main, CONFIG_LOG_DEFAULT_LEVEL);
 #error No camera chosen in devicetree. Missing "--shield" or "--snippet video-sw-generator" flag?
 #endif
 
+#define NUM_BUFS 2
+
+int __weak video_transform_setup(struct video_format in_fmt, struct video_format *out_fmt)
+{
+	*out_fmt = in_fmt;
+
+	return 0;
+}
+
+int __weak video_transform_process(struct video_buffer *in_buf, struct video_buffer **out_buf)
+{
+	*out_buf = in_buf;
+
+	return 0;
+}
+
 #if DT_HAS_CHOSEN(zephyr_display)
 static inline int display_setup(const struct device *const display_dev, const uint32_t pixfmt)
 {
@@ -91,10 +107,11 @@ static inline void video_display_frame(const struct device *const display_dev,
 
 int main(void)
 {
-	struct video_buffer *buffers[CONFIG_VIDEO_BUFFER_POOL_NUM_MAX];
 	struct video_buffer *vbuf = &(struct video_buffer){};
+	struct video_buffer *transformed_buf = &(struct video_buffer){};
 	const struct device *video_dev;
 	struct video_format fmt;
+	struct video_format transformed_fmt;
 	struct video_caps caps;
 	struct video_frmival frmival;
 	struct video_frmival_enum fie;
@@ -262,6 +279,13 @@ int main(void)
 		tp_set_ret = video_set_ctrl(video_dev, &ctrl);
 	}
 
+	/* Set up transformation */
+	err = video_transform_setup(fmt, &transformed_fmt);
+	if (err) {
+		LOG_ERR("Unable to set up transformation");
+		return 0;
+	}
+
 #if DT_HAS_CHOSEN(zephyr_display)
 	const struct device *const display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display));
 
@@ -270,7 +294,7 @@ int main(void)
 		return 0;
 	}
 
-	err = display_setup(display_dev, fmt.pixelformat);
+	err = display_setup(display_dev, transformed_fmt.pixelformat);
 	if (err) {
 		LOG_ERR("Unable to set up display");
 		return err;
@@ -285,19 +309,19 @@ int main(void)
 	}
 
 	/* Alloc video buffers and enqueue for capture */
-	for (i = 0; i < ARRAY_SIZE(buffers); i++) {
+	for (i = 0; i < NUM_BUFS; i++) {
 		/*
 		 * For some hardwares, such as the PxP used on i.MX RT1170 to do image rotation,
 		 * buffer alignment is needed in order to achieve the best performance
 		 */
-		buffers[i] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN,
+		vbuf = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN,
 							K_FOREVER);
-		if (buffers[i] == NULL) {
+		if (vbuf == NULL) {
 			LOG_ERR("Unable to alloc video buffer");
 			return 0;
 		}
-		buffers[i]->type = type;
-		video_enqueue(video_dev, buffers[i]);
+		vbuf->type = type;
+		video_enqueue(video_dev, vbuf);
 	}
 
 	/* Start video capture */
@@ -328,8 +352,15 @@ int main(void)
 		}
 #endif
 
+		/* Do transformation */
+		err = video_transform_process(vbuf, &transformed_buf);
+		if (err) {
+			LOG_ERR("Unable to transform video frame");
+			return 0;
+		}
+
 #if DT_HAS_CHOSEN(zephyr_display)
-		video_display_frame(display_dev, vbuf, fmt);
+		video_display_frame(display_dev, transformed_buf, transformed_fmt);
 #endif
 
 		err = video_enqueue(video_dev, vbuf);