Merge pull request opencv#18535 from joshdoe:gray16_gstreamer_writing

Add CV_16UC1/GRAY16_LE support to GStreamer backend for VideoWriter

* videoio(backend): add Writer_open_with_params to plugin API

This will allow arbitrary parameters to be passed to plugin backends

* videoio(gstreamer): add GRAY16_LE/CV_16UC1 writing support to GStreamer

This introduces a new property VIDEOWRITER_PROP_DEPTH, which defaults to
CV_8U, but for GStreamer can be set to CV_16U.

Also, fix another test to not fail if plugin isn't found, copying logic
from the read_write test.

* videoio(plugin): fix handling plugins with previous API level

* videoio: coding style

* fix warning

Author: joshdoe

modules/videoio/include/opencv2/videoio.hpp

@@ -194,8 +194,9 @@ enum VideoWriterProperties {
   VIDEOWRITER_PROP_QUALITY = 1,    //!< Current quality (0..100%) of the encoded videostream. Can be adjusted dynamically in some codecs.
   VIDEOWRITER_PROP_FRAMEBYTES = 2, //!< (Read-only): Size of just encoded video frame. Note that the encoding order may be different from representation order.
   VIDEOWRITER_PROP_NSTRIPES = 3,   //!< Number of stripes for parallel encoding. -1 for auto detection.
-  VIDEOWRITER_PROP_IS_COLOR = 4    //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
+  VIDEOWRITER_PROP_IS_COLOR = 4,   //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
                                    //!< will work with grayscale frames.
+  VIDEOWRITER_PROP_DEPTH = 5       //!< Defaults to CV_8U.
 };
 
 //! @} videoio_flags_base

modules/videoio/src/backend_plugin.cpp

@@ -205,10 +205,15 @@ class PluginBackend: public IBackend
         FN_opencv_videoio_plugin_init_t fn_init = reinterpret_cast<FN_opencv_videoio_plugin_init_t>(lib_->getSymbol(init_name));
         if (fn_init)
         {
-            plugin_api_ = fn_init(ABI_VERSION, API_VERSION, NULL);
+            for (int supported_api_version = API_VERSION; supported_api_version >= 0; supported_api_version--)
+            {
+                plugin_api_ = fn_init(ABI_VERSION, supported_api_version, NULL);
+                if (plugin_api_)
+                    break;
+            }
             if (!plugin_api_)
             {
-                CV_LOG_INFO(NULL, "Video I/O: plugin is incompatible: " << lib->getName());
+                CV_LOG_INFO(NULL, "Video I/O: plugin is incompatible (can't be initialized): " << lib->getName());
                 return;
             }
             if (plugin_api_->api_header.opencv_version_major != CV_VERSION_MAJOR)
@@ -232,8 +237,29 @@ class PluginBackend: public IBackend
                 plugin_api_ = NULL;
                 return;
             }
-            // TODO Preview: add compatibility API/ABI checks
-            CV_LOG_INFO(NULL, "Video I/O: loaded plugin '" << plugin_api_->api_header.api_description << "'");
+            CV_LOG_INFO(NULL, "Video I/O: initialized '" << plugin_api_->api_header.api_description << "': built with "
+                << cv::format("OpenCV %d.%d (ABI/API = %d/%d)",
+                     plugin_api_->api_header.opencv_version_major, plugin_api_->api_header.opencv_version_minor,
+                     plugin_api_->api_header.min_api_version, plugin_api_->api_header.api_version)
+                << ", current OpenCV version is '" CV_VERSION "' (ABI/API = " << ABI_VERSION << "/" << API_VERSION << ")"
+            );
+            if (plugin_api_->api_header.min_api_version != ABI_VERSION)  // future: range can be here
+            {
+                // actually this should never happen due to checks in plugin's init() function
+                CV_LOG_ERROR(NULL, "Video I/O: plugin is not supported due to incompatible ABI = " << plugin_api_->api_header.min_api_version);
+                plugin_api_ = NULL;
+                return;
+            }
+            if (plugin_api_->api_header.api_version != API_VERSION)
+            {
+                CV_LOG_INFO(NULL, "Video I/O: NOTE: plugin is supported, but there is API version mismath: "
+                    << cv::format("plugin API level (%d) != OpenCV API level (%d)", plugin_api_->api_header.api_version, API_VERSION));
+                if (plugin_api_->api_header.api_version < API_VERSION)
+                {
+                    CV_LOG_INFO(NULL, "Video I/O: NOTE: some functionality may be unavailable due to lack of support by plugin implementation");
+                }
+            }
+            CV_LOG_INFO(NULL, "Video I/O: plugin is ready to use '" << plugin_api_->api_header.api_description << "'");
         }
         else
         {
@@ -508,11 +534,31 @@ class PluginWriter : public cv::IVideoWriter
     {
         CV_Assert(plugin_api);
         CvPluginWriter writer = NULL;
-        if (plugin_api->Writer_open)
+        if (plugin_api->api_header.api_version >= 1 && plugin_api->Writer_open_with_params)
+        {
+            CV_Assert(plugin_api->Writer_release);
+            CV_Assert(!filename.empty());
+            std::vector<int> vint_params = params.getIntVector();
+            int* c_params = &vint_params[0];
+            unsigned n_params = (unsigned)(vint_params.size() / 2);
+
+            if (CV_ERROR_OK == plugin_api->Writer_open_with_params(filename.c_str(), fourcc, fps, sz.width, sz.height, c_params, n_params, &writer))
+            {
+                CV_Assert(writer);
+                return makePtr<PluginWriter>(plugin_api, writer);
+            }
+        }
+        else if (plugin_api->Writer_open)
         {
             CV_Assert(plugin_api->Writer_release);
             CV_Assert(!filename.empty());
             const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
+            const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U);
+            if (depth != CV_8U)
+            {
+                CV_LOG_WARNING(NULL, "Video I/O plugin doesn't support (due to lower API level) creation of VideoWriter with depth != CV_8U");
+                return Ptr<PluginWriter>();
+            }
             if (CV_ERROR_OK == plugin_api->Writer_open(filename.c_str(), fourcc, fps, sz.width, sz.height, isColor, &writer))
             {
                 CV_Assert(writer);

modules/videoio/src/cap_ffmpeg.cpp

@@ -396,7 +396,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char
 static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
 {
     {
-        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
+        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
         CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
         "FFmpeg OpenCV Video I/O plugin"
     },
@@ -411,7 +411,8 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
     /*  9*/cv_writer_release,
     /* 10*/cv_writer_get_prop,
     /* 11*/cv_writer_set_prop,
-    /* 12*/cv_writer_write
+    /* 12*/cv_writer_write,
+    /* 13 Writer_open_with_params*/NULL
 };
 
 } // namespace

modules/videoio/src/cap_gstreamer.cpp

@@ -1240,8 +1240,8 @@ class CvVideoWriter_GStreamer : public CvVideoWriter
 {
 public:
     CvVideoWriter_GStreamer()
-        : input_pix_fmt(0),
-          num_frames(0), framerate(0)
+        : ipl_depth(CV_8U)
+        , input_pix_fmt(0), num_frames(0), framerate(0)
     {
     }
     virtual ~CvVideoWriter_GStreamer() CV_OVERRIDE
@@ -1263,14 +1263,16 @@ class CvVideoWriter_GStreamer : public CvVideoWriter
     int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_GSTREAMER; }
 
     bool open(const std::string &filename, int fourcc,
-                       double fps, const Size &frameSize, bool isColor );
+                       double fps, const Size &frameSize, bool isColor, int depth );
     void close();
     bool writeFrame( const IplImage* image ) CV_OVERRIDE;
+
+    int getIplDepth() const { return ipl_depth; }
 protected:
     const char* filenameToMimetype(const char* filename);
     GSafePtr<GstElement> pipeline;
     GSafePtr<GstElement> source;
-
+    int ipl_depth;
     int input_pix_fmt;
     int num_frames;
     double framerate;
@@ -1396,6 +1398,7 @@ const char* CvVideoWriter_GStreamer::filenameToMimetype(const char *filename)
  * \param fps desired framerate
  * \param frameSize the size of the expected frames
  * \param is_color color or grayscale
+ * \param depth the depth of the expected frames
  * \return success
  *
  * We support 2 modes of operation. Either the user enters a filename and a fourcc
@@ -1408,7 +1411,8 @@ const char* CvVideoWriter_GStreamer::filenameToMimetype(const char *filename)
  *
  */
 bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
-                                    double fps, const cv::Size &frameSize, bool is_color )
+                                    double fps, const cv::Size &frameSize,
+                                    bool is_color, int depth )
 {
     // check arguments
     CV_Assert(!filename.empty());
@@ -1548,6 +1552,8 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
 
     if (fourcc == CV_FOURCC('M','J','P','G') && frameSize.height == 1)
     {
+        CV_Assert(depth == CV_8U);
+        ipl_depth = IPL_DEPTH_8U;
         input_pix_fmt = GST_VIDEO_FORMAT_ENCODED;
         caps.attach(gst_caps_new_simple("image/jpeg",
                                         "framerate", GST_TYPE_FRACTION, int(fps_num), int(fps_denom),
@@ -1556,6 +1562,8 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
     }
     else if (is_color)
     {
+        CV_Assert(depth == CV_8U);
+        ipl_depth = IPL_DEPTH_8U;
         input_pix_fmt = GST_VIDEO_FORMAT_BGR;
         bufsize = frameSize.width * frameSize.height * 3;
 
@@ -1569,8 +1577,9 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
         caps.attach(gst_caps_fixate(caps.detach()));
         CV_Assert(caps);
     }
-    else
+    else if (!is_color && depth == CV_8U)
     {
+        ipl_depth = IPL_DEPTH_8U;
         input_pix_fmt = GST_VIDEO_FORMAT_GRAY8;
         bufsize = frameSize.width * frameSize.height;
 
@@ -1582,6 +1591,26 @@ bool CvVideoWriter_GStreamer::open( const std::string &filename, int fourcc,
                                         NULL));
         caps.attach(gst_caps_fixate(caps.detach()));
     }
+    else if (!is_color && depth == CV_16U)
+    {
+        ipl_depth = IPL_DEPTH_16U;
+        input_pix_fmt = GST_VIDEO_FORMAT_GRAY16_LE;
+        bufsize = frameSize.width * frameSize.height * 2;
+
+        caps.attach(gst_caps_new_simple("video/x-raw",
+                                        "format", G_TYPE_STRING, "GRAY16_LE",
+                                        "width", G_TYPE_INT, frameSize.width,
+                                        "height", G_TYPE_INT, frameSize.height,
+                                        "framerate", GST_TYPE_FRACTION, gint(fps_num), gint(fps_denom),
+                                        NULL));
+        caps.attach(gst_caps_fixate(caps.detach()));
+    }
+    else
+    {
+        CV_WARN("unsupported depth=" << depth <<", and is_color=" << is_color << " combination");
+        pipeline.release();
+        return false;
+    }
 
     gst_app_src_set_caps(GST_APP_SRC(source.get()), caps);
     gst_app_src_set_stream_type(GST_APP_SRC(source.get()), GST_APP_STREAM_TYPE_STREAM);
@@ -1659,6 +1688,12 @@ bool CvVideoWriter_GStreamer::writeFrame( const IplImage * image )
             return false;
         }
     }
+    else if (input_pix_fmt == GST_VIDEO_FORMAT_GRAY16_LE) {
+        if (image->nChannels != 1 || image->depth != IPL_DEPTH_16U) {
+            CV_WARN("cvWriteFrame() needs images with depth = IPL_DEPTH_16U and nChannels = 1.");
+            return false;
+        }
+    }
     else {
         CV_WARN("cvWriteFrame() needs BGR or grayscale images\n");
         return false;
@@ -1699,9 +1734,10 @@ Ptr<IVideoWriter> create_GStreamer_writer(const std::string& filename, int fourc
 {
     CvVideoWriter_GStreamer* wrt = new CvVideoWriter_GStreamer;
     const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
+    const int depth = params.get(VIDEOWRITER_PROP_DEPTH, CV_8U);
     try
     {
-        if (wrt->open(filename, fourcc, fps, frameSize, isColor))
+        if (wrt->open(filename, fourcc, fps, frameSize, isColor, depth))
             return makePtr<LegacyWriter>(wrt);
         delete wrt;
     }
@@ -1921,15 +1957,40 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
 }
 
 static
-CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
-                                    CV_OUT CvPluginWriter* handle)
+CvResult CV_API_CALL cv_writer_open_with_params(
+        const char* filename, int fourcc, double fps, int width, int height,
+        int* params, unsigned n_params,
+        CV_OUT CvPluginWriter* handle)
 {
     CvVideoWriter_GStreamer* wrt = 0;
     try
     {
-        wrt = new CvVideoWriter_GStreamer();
         CvSize sz = { width, height };
-        if(wrt && wrt->open(filename, fourcc, fps, sz, isColor))
+        bool isColor = true;
+        int depth = CV_8U;
+        if (params)
+        {
+            for (unsigned i = 0; i < n_params; ++i)
+            {
+                const int prop = params[i*2];
+                const int value = params[i*2 + 1];
+                switch (prop)
+                {
+                case VIDEOWRITER_PROP_IS_COLOR:
+                    isColor = value != 0;
+                    break;
+                case VIDEOWRITER_PROP_DEPTH:
+                    depth = value;
+                    break;
+                default:
+                    // TODO emit message about non-recognized propert
+                    // FUTURE: there should be mandatory and optional properties
+                    return CV_ERROR_FAIL;
+                }
+            }
+        }
+        wrt = new CvVideoWriter_GStreamer();
+        if (wrt && wrt->open(filename, fourcc, fps, sz, isColor, depth))
         {
             *handle = (CvPluginWriter)wrt;
             return CV_ERROR_OK;
@@ -1943,6 +2004,14 @@ CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps
     return CV_ERROR_FAIL;
 }
 
+static
+CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
+    CV_OUT CvPluginWriter* handle)
+{
+    int params[2] = { VIDEOWRITER_PROP_IS_COLOR, isColor };
+    return cv_writer_open_with_params(filename, fourcc, fps, width, height, params, 1, handle);
+}
+
 static
 CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle)
 {
@@ -1975,7 +2044,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char
         CvVideoWriter_GStreamer* instance = (CvVideoWriter_GStreamer*)handle;
         CvSize sz = { width, height };
         IplImage img;
-        cvInitImageHeader(&img, sz, IPL_DEPTH_8U, cn);
+        cvInitImageHeader(&img, sz, instance->getIplDepth(), cn);
         cvSetData(&img, const_cast<unsigned char*>(data), step);
         return instance->writeFrame(&img) ? CV_ERROR_OK : CV_ERROR_FAIL;
     }
@@ -2003,7 +2072,8 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
     /*  9*/cv_writer_release,
     /* 10*/cv_writer_get_prop,
     /* 11*/cv_writer_set_prop,
-    /* 12*/cv_writer_write
+    /* 12*/cv_writer_write,
+    /* 13*/cv_writer_open_with_params
 };
 
 } // namespace
@@ -2012,7 +2082,7 @@ const OpenCV_VideoIO_Plugin_API_preview* opencv_videoio_plugin_init_v0(int reque
 {
     if (requested_abi_version != 0)
         return NULL;
-    if (requested_api_version != 0)
+    if (requested_api_version != 0 && requested_api_version != 1)
         return NULL;
     return &cv::plugin_api_v0;
 }

modules/videoio/src/cap_interface.hpp

@@ -116,6 +116,18 @@ class VideoWriterParameters
         }
         return unusedParams;
     }
+
+    std::vector<int> getIntVector() const
+    {
+        std::vector<int> vint_params;
+        for (const auto& param : params_)
+        {
+            vint_params.push_back(param.key);
+            vint_params.push_back(param.value);
+        }
+        return vint_params;
+    }
+
 private:
     std::vector<VideoWriterParameter> params_;
 };

modules/videoio/src/cap_mfx_plugin.cpp

@@ -188,7 +188,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char
 static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
 {
     {
-        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
+        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
         CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
         "MediaSDK OpenCV Video I/O plugin"
     },
@@ -203,7 +203,8 @@ static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
     /*  9*/cv_writer_release,
     /* 10*/cv_writer_get_prop,
     /* 11*/cv_writer_set_prop,
-    /* 12*/cv_writer_write
+    /* 12*/cv_writer_write,
+    /* 13 Writer_open_with_params*/NULL
 };
 
 } // namespace

modules/videoio/src/cap_msmf.cpp

@@ -1866,7 +1866,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter handle, const unsigned char*
 static const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
 {
     {
-        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
+        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
         CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
         "Microsoft Media Foundation OpenCV Video I/O plugin"
     },

modules/videoio/src/cap_ueye.cpp

@@ -467,7 +467,7 @@ CvResult CV_API_CALL cv_writer_write(CvPluginWriter /*handle*/, const unsigned c
 const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
 {
     {
-        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, API_VERSION,
+        sizeof(OpenCV_VideoIO_Plugin_API_preview), ABI_VERSION, 0/*API_VERSION*/,
         CV_VERSION_MAJOR, CV_VERSION_MINOR, CV_VERSION_REVISION, CV_VERSION_STATUS,
         "uEye OpenCV Video I/O plugin"
     },
@@ -482,7 +482,8 @@ const OpenCV_VideoIO_Plugin_API_preview plugin_api_v0 =
     /*  9*/cv_writer_release,
     /* 10*/cv_writer_get_prop,
     /* 11*/cv_writer_set_prop,
-    /* 12*/cv_writer_write
+    /* 12*/cv_writer_write,
+    /* 13 Writer_open_with_params*/NULL
 };
 } // namespace
 } // namespace cv