Merge pull request opencv#19516 from dbudniko:dbudniko/gapi_render_frame_cpu

Dmitry Budnikov · web-flow · commit 0cdbdfdf9fe6 · 2021-02-15T19:57:22.000Z
Render Frame NV12 on CPU

* render frame on CPU

* doxygen fix

* address review from Alexey

* comment fixes

* Address comments from Ruslan

* remove NV12 specific

* mat clone and more renaming

* misprint fix
diff --git a/modules/gapi/include/opencv2/gapi/render/render.hpp b/modules/gapi/include/opencv2/gapi/render/render.hpp
@@ -97,6 +97,17 @@ void GAPI_EXPORTS render(cv::Mat& y_plane,
                          const Prims& prims,
                          cv::GCompileArgs&& args = {});
 
+/** @brief The function renders on the input media frame passed drawing primitivies
+
+@param frame input Media Frame :  @ref cv::MediaFrame.
+@param prims vector of drawing primitivies
+@param args graph compile time parameters
+*/
+void GAPI_EXPORTS render(cv::MediaFrame& frame,
+                         const Prims& prims,
+                         cv::GCompileArgs&& args = {});
+
+
 G_TYPED_KERNEL_M(GRenderNV12, <GMat2(cv::GMat,cv::GMat,cv::GArray<wip::draw::Prim>)>, "org.opencv.render.nv12")
 {
      static GMatDesc2 outMeta(GMatDesc y_plane, GMatDesc uv_plane, GArrayDesc)
@@ -113,6 +124,14 @@ G_TYPED_KERNEL(GRenderBGR, <cv::GMat(cv::GMat,cv::GArray<wip::draw::Prim>)>, "or
      }
 };
 
+G_TYPED_KERNEL(GRenderFrame, <cv::GFrame(cv::GFrame, cv::GArray<wip::draw::Prim>)>, "org.opencv.render.frame")
+{
+    static GFrameDesc outMeta(GFrameDesc desc, GArrayDesc)
+    {
+        return desc;
+    }
+};
+
 /** @brief Renders on 3 channels input
 
 Output image must be 8-bit unsigned planar 3-channel image
@@ -134,6 +153,17 @@ uv image must be 8-bit unsigned planar 2-channel image @ref CV_8UC2
 GAPI_EXPORTS GMat2 renderNV12(const GMat& y,
                               const GMat& uv,
                               const GArray<Prim>& prims);
+
+/** @brief Renders Media Frame
+
+Output media frame frame cv::MediaFrame
+
+@param m_frame input image: cv::MediaFrame @ref cv::MediaFrame
+@param prims draw primitives
+*/
+GAPI_EXPORTS GFrame renderFrame(const GFrame& m_frame,
+                                const GArray<Prim>& prims);
+
 //! @} gapi_draw_api
 
 } // namespace draw
diff --git a/modules/gapi/src/api/render.cpp b/modules/gapi/src/api/render.cpp
@@ -33,6 +33,20 @@ void cv::gapi::wip::draw::render(cv::Mat& y_plane,
                cv::gout(y_plane, uv_plane), std::move(args));
 }
 
+void cv::gapi::wip::draw::render(cv::MediaFrame& frame,
+                                 const Prims& prims,
+                                 cv::GCompileArgs&& args)
+{
+    cv::GFrame in, out;
+    cv::GArray<cv::gapi::wip::draw::Prim> arr;
+    out = cv::gapi::wip::draw::renderFrame(in, arr);
+
+    cv::GComputation comp(cv::GIn(in, arr), cv::GOut(out));
+    comp.apply(cv::gin(frame, prims),
+               cv::gout(frame), std::move(args));
+}
+
+
 void cv::gapi::wip::draw::cvtYUVToNV12(const cv::Mat& yuv,
                                        cv::Mat& y,
                                        cv::Mat& uv)
@@ -69,3 +83,9 @@ cv::gapi::wip::draw::renderNV12(const cv::GMat& y,
 {
     return cv::gapi::wip::draw::GRenderNV12::on(y, uv, prims);
 }
+
+cv::GFrame cv::gapi::wip::draw::renderFrame(const cv::GFrame& frame,
+                                                const cv::GArray<cv::gapi::wip::draw::Prim>& prims)
+{
+    return cv::gapi::wip::draw::GRenderFrame::on(frame, prims);
+}
diff --git a/modules/gapi/src/backends/render/grenderocv.cpp b/modules/gapi/src/backends/render/grenderocv.cpp
@@ -114,8 +114,83 @@ GAPI_OCV_KERNEL_ST(RenderNV12OCVImpl, cv::gapi::wip::draw::GRenderNV12, RenderOC
     }
 };
 
+GAPI_OCV_KERNEL_ST(RenderFrameOCVImpl, cv::gapi::wip::draw::GRenderFrame, RenderOCVState)
+{
+    static void run(const cv::MediaFrame & in,
+                    const cv::gapi::wip::draw::Prims & prims,
+                    cv::MediaFrame & out,
+                    RenderOCVState & state)
+    {
+        GAPI_Assert(in.desc().fmt == cv::MediaFormat::NV12);
+
+        // FIXME: consider a better approach (aka native inplace operation)
+        // Non-intuitive logic with shared_ptr Priv class
+        out = in;
+
+        auto desc = out.desc();
+        auto w_out = out.access(cv::MediaFrame::Access::W);
+
+        auto out_y = cv::Mat(desc.size, CV_8UC1, w_out.ptr[0], w_out.stride[0]);
+        auto out_uv = cv::Mat(desc.size / 2, CV_8UC2, w_out.ptr[1], w_out.stride[1]);
+
+        auto r_in = in.access(cv::MediaFrame::Access::R);
+
+        auto in_y = cv::Mat(desc.size, CV_8UC1, r_in.ptr[0], r_in.stride[0]);
+        auto in_uv = cv::Mat(desc.size / 2, CV_8UC2, r_in.ptr[1], r_in.stride[1]);
+
+        /* FIXME How to render correctly on NV12 format ?
+         *
+         * Rendering on NV12 via OpenCV looks like this:
+         *
+         * y --------> 1)(NV12 -> YUV) -> yuv -> 2)draw -> yuv -> 3)split -------> out_y
+         *                  ^                                         |
+         *                  |                                         |
+         * uv --------------                                          `----------> out_uv
+         *
+         *
+         * 1) Collect yuv mat from two planes, uv plain in two times less than y plane
+         *    so, upsample uv in two times, with bilinear interpolation
+         *
+         * 2) Render primitives on YUV
+         *
+         * 3) Convert yuv to NV12 (using bilinear interpolation)
+         *
+         */
+
+         // NV12 -> YUV
+        cv::Mat upsample_uv, yuv;
+        cv::resize(in_uv, upsample_uv, in_uv.size() * 2, cv::INTER_LINEAR);
+        cv::merge(std::vector<cv::Mat>{in_y, upsample_uv}, yuv);
+
+        cv::gapi::wip::draw::drawPrimitivesOCVYUV(yuv, prims, state.ftpr);
+
+        // YUV -> NV12
+        cv::Mat out_u, out_v, uv_plane;
+        std::vector<cv::Mat> chs = { out_y, out_u, out_v };
+        cv::split(yuv, chs);
+        cv::merge(std::vector<cv::Mat>{chs[1], chs[2]}, uv_plane);
+        cv::resize(uv_plane, out_uv, uv_plane.size() / 2, cv::INTER_LINEAR);
+    }
+
+    static void setup(const cv::GFrameDesc&   /* in_nv12  */,
+        const cv::GArrayDesc& /* prims */,
+        std::shared_ptr<RenderOCVState>&state,
+        const cv::GCompileArgs & args)
+    {
+        using namespace cv::gapi::wip::draw;
+        auto has_freetype_font = cv::gapi::getCompileArg<freetype_font>(args);
+        state = std::make_shared<RenderOCVState>();
+
+        if (has_freetype_font)
+        {
+            state->ftpr = std::make_shared<FTTextRender>(has_freetype_font->path);
+        }
+    }
+};
+
+
 cv::gapi::GKernelPackage cv::gapi::render::ocv::kernels()
 {
-    const static auto pkg = cv::gapi::kernels<RenderBGROCVImpl, RenderNV12OCVImpl>();
+    const static auto pkg = cv::gapi::kernels<RenderBGROCVImpl, RenderNV12OCVImpl, RenderFrameOCVImpl>();
     return pkg;
 }
diff --git a/modules/gapi/test/common/gapi_render_tests.hpp b/modules/gapi/test/common/gapi_render_tests.hpp
@@ -130,6 +130,8 @@ struct Fixture : public RenderBGRTestBase API {                  \
 #define GAPI_RENDER_TEST_FIXTURES(Fixture, API, Number, ...)                    \
     GAPI_RENDER_TEST_FIXTURE_BGR(RenderBGR##Fixture,   GET_VA_ARGS(API), Number, __VA_ARGS__) \
     GAPI_RENDER_TEST_FIXTURE_NV12(RenderNV12##Fixture, GET_VA_ARGS(API), Number, __VA_ARGS__) \
+    GAPI_RENDER_TEST_FIXTURE_NV12(RenderMFrame##Fixture, GET_VA_ARGS(API), Number, __VA_ARGS__) \
+
 
 using Points = std::vector<cv::Point>;
 GAPI_RENDER_TEST_FIXTURES(TestTexts,     FIXTURE_API(std::string, cv::Point, double, cv::Scalar), 4, text, org, fs, color)
diff --git a/modules/gapi/test/render/gapi_render_tests_ocv.cpp b/modules/gapi/test/render/gapi_render_tests_ocv.cpp
