Merge pull request opencv#19460 from mikhail-nikolskiy:videoio-hw

videoio: HW decode/encode in FFMPEG backend; new properties with support in FFMPEG/GST/MSMF

* HW acceleration in FFMPEG backend

* fixes on Windows, remove D3D9

* HW acceleration in FFMPEG backend

* fixes on Windows, remove D3D9

* improve va test

* Copyright

* check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+

* CAP_MSMF test on .mp4

* .mp4 in test

* improve va test

* Copyright

* check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+

* CAP_MSMF test on .mp4

* .mp4 in test

* .avi for GStreamer test

* revert changes around seek()

* cv_writer_open_with_params

* params.warnUnusedParameters

* VideoCaptureParameters in GStreamer

* open_with_params

* params->getUnused

* Reduce PSNR threshold 33->32 (other tests use 30)

* require FFMPEG 4.0+; PSNR 30 as in other tests

* GStreamer AVI-demux plugin not installed in Ubuntu test environment?

* fix build on very old ffmpeg

* fix build on very old ffmpeg

* fix build issues

* fix build issues (static_cast)

* FFMPEG built on Windows without H264 encoder?

* fix for write_nothing test on VAAPI

* fix warnings

* fix cv_writer_get_prop in plugins

* use avcodec_get_hw_frames_parameters; more robust fallback to SW codecs

* internal function hw_check_device() for device check/logging

* two separate tests for HW read and write

* image size 640x480 in encode test

* WITH_VA=ON (only .h headers used in OpenCV, no linkage dependency)

* exception on VP9 SW encoder?

* rebase master; refine info message

* videoio: fix FFmpeg standalone plugin build

* videoio(ffmpeg): eliminate MSVC build warnings

* address review comments

* videoio(hw): update videocapture_acceleration.read test

- remove parallel decoding by SW code path
- check PSNR against the original generated image

* videoio: minor fixes

* videoio(test): disable unsupported MSMF cases (SW and HW)

* videoio(test): update PSNR thresholds for HW acceleration read

* videoio(test): update debug messages

* "hw_acceleration" whitelisting parameter

* little optimization in test

* D3D11VA supports decoders, doesn't support encoders

* videoio(test): adjust PSNR threshold in write_read_position tests

* videoio(ffmpeg): fix rejecting on acceleration device name mismatch

* videoio(ffmpeg): fix compilation USE_AV_HW_CODECS=0, add more debug logging

* videoio: rework VideoAccelerationType behavior

- enum is not a bitset
- default value is backend specific
- only '_NONE' and '_ANY' may fallback on software processing
- specific H/W acceleration doesn't fallback on software processing. It fails if there is no support for specified H/W acceleration.

* videoio(test): fix for current FFmpeg wrapper

Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>

Author: mikhail-nikolskiy

modules/videoio/include/opencv2/videoio.hpp

@@ -78,7 +78,7 @@ namespace cv
 //! @{
 
 
-/** @brief %VideoCapture API backends identifier.
+/** @brief cv::VideoCapture API backends identifier.
 
 Select preferred API for a capture object.
 To be used in the VideoCapture::VideoCapture() constructor or VideoCapture::open()
@@ -124,7 +124,7 @@ enum VideoCaptureAPIs {
        CAP_UEYE         = 2500,         //!< uEye Camera API
      };
 
-/** @brief %VideoCapture generic properties identifier.
+/** @brief cv::VideoCapture generic properties identifier.
 
  Reading / writing properties involves many layers. Some unexpected result might happens along this chain.
  Effective behaviour depends from device hardware, driver and API Backend.
@@ -182,12 +182,14 @@ enum VideoCaptureProperties {
        CAP_PROP_BITRATE       =47, //!< (read-only) Video bitrate in kbits/s
        CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only)
        CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata  (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499)
+       CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific.
+       CAP_PROP_HW_DEVICE      =51, //!< (**open-only**) Hardware device index (select GPU if multiple available)
 #ifndef CV_DOXYGEN
        CV__CAP_PROP_LATEST
 #endif
      };
 
-/** @brief %VideoWriter generic properties identifier.
+/** @brief cv::VideoWriter generic properties identifier.
  @sa VideoWriter::get(), VideoWriter::set()
 */
 enum VideoWriterProperties {
@@ -196,14 +198,39 @@ enum VideoWriterProperties {
   VIDEOWRITER_PROP_NSTRIPES = 3,   //!< Number of stripes for parallel encoding. -1 for auto detection.
   VIDEOWRITER_PROP_IS_COLOR = 4,   //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
                                    //!< will work with grayscale frames.
-  VIDEOWRITER_PROP_DEPTH = 5       //!< Defaults to CV_8U.
+  VIDEOWRITER_PROP_DEPTH = 5,      //!< Defaults to CV_8U.
+  VIDEOWRITER_PROP_HW_ACCELERATION = 6, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in VideoWriter constructor / .open() method. Default value is backend-specific.
+  VIDEOWRITER_PROP_HW_DEVICE       = 7, //!< (**open-only**) Hardware device index (select GPU if multiple available)
+#ifndef CV_DOXYGEN
+  CV__VIDEOWRITER_PROP_LATEST
+#endif
 };
 
 //! @} videoio_flags_base
 
 //! @addtogroup videoio_flags_others
 //! @{
 
+/** @brief Video Acceleration type
+ *
+ * Used as value in #CAP_PROP_HW_ACCELERATION and #VIDEOWRITER_PROP_HW_ACCELERATION
+ *
+ * @note In case of FFmpeg backend, it translated to enum AVHWDeviceType (https://github.com/FFmpeg/FFmpeg/blob/master/libavutil/hwcontext.h)
+ */
+enum VideoAccelerationType
+{
+    VIDEO_ACCELERATION_NONE     =  0,  //!< Do not require any specific H/W acceleration, prefer software processing.
+                                       //!< Reading of this value means that special H/W accelerated handling is not added or not detected by OpenCV.
+
+    VIDEO_ACCELERATION_ANY      =  1,  //!< Prefer to use H/W acceleration. If no one supported, then fallback to software processing.
+                                       //!< @note H/W acceleration may require special configuration of used environment.
+                                       //!< @note Results in encoding scenario may differ between software and hardware accelerated encoders.
+
+    VIDEO_ACCELERATION_D3D11    =  2,  //!< DirectX 11
+    VIDEO_ACCELERATION_VAAPI    =  3,  //!< VAAPI
+    VIDEO_ACCELERATION_MFX      =  4,  //!< libmfx (Intel MediaSDK/oneVPL)
+};
+
 /** @name IEEE 1394 drivers
     @{
 */

modules/videoio/src/backend_plugin.cpp

@@ -415,7 +415,7 @@ class PluginCapture : public cv::IVideoCapture
         if (plugin_api->api_header.api_version >= 1 && plugin_api->v1.Capture_open_with_params)
         {
             std::vector<int> vint_params = params.getIntVector();
-            int* c_params = &vint_params[0];
+            int* c_params = vint_params.data();
             unsigned n_params = (unsigned)(vint_params.size() / 2);
 
             if (CV_ERROR_OK == plugin_api->v1.Capture_open_with_params(
@@ -547,7 +547,7 @@ class PluginWriter : public cv::IVideoWriter
             }
             if (params.warnUnusedParameters())
             {
-                CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in VideoWriter, see logger INFO channel for details");
+                CV_LOG_ERROR(NULL, "VIDEOIO: unsupported parameters in VideoWriter, see logger INFO channel for details");
                 return Ptr<PluginWriter>();
             }
             if (CV_ERROR_OK == plugin_api->v0.Writer_open(filename.c_str(), fourcc, fps, sz.width, sz.height, isColor, &writer))

modules/videoio/src/backend_static.cpp

@@ -19,16 +19,17 @@ void applyParametersFallback(const Ptr<IVideoCapture>& cap, const VideoCapturePa
     {
         double value = params.get<double>(prop, -1);
         CV_LOG_INFO(NULL, "VIDEOIO: apply parameter: [" << prop << "]=" <<
-                          cv::format("%g / %lld / 0x%16llx", value, (long long)value, (long long)value));
+                          cv::format("%g / %lld / 0x%016llx", value, (long long)value, (long long)value));
         if (!cap->setProperty(prop, value))
         {
-            CV_Error_(cv::Error::StsNotImplemented, ("VIDEOIO: Failed to apply invalid or unsupported parameter: [%d]=%g / %lld / 0x%08llx", prop, value, (long long)value, (long long)value));
+            if (prop != CAP_PROP_HW_ACCELERATION && prop != CAP_PROP_HW_DEVICE) { // optional parameters
+                CV_Error_(cv::Error::StsNotImplemented, ("VIDEOIO: Failed to apply invalid or unsupported parameter: [%d]=%g / %lld / 0x%08llx", prop, value, (long long)value, (long long)value));
+            }
         }
     }
     // NB: there is no dedicated "commit" parameters event, implementations should commit after each property automatically
 }
 
-
 // Legacy API. Modern API with parameters is below
 class StaticBackend: public IBackend
 {

modules/videoio/src/cap_ffmpeg.cpp

@@ -165,7 +165,7 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
 {
 public:
     CvVideoWriter_FFMPEG_proxy() { ffmpegWriter = 0; }
-    CvVideoWriter_FFMPEG_proxy(const cv::String& filename, int fourcc, double fps, cv::Size frameSize, bool isColor) { ffmpegWriter = 0; open(filename, fourcc, fps, frameSize, isColor); }
+    CvVideoWriter_FFMPEG_proxy(const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params) { ffmpegWriter = 0; open(filename, fourcc, fps, frameSize, params); }
     virtual ~CvVideoWriter_FFMPEG_proxy() { close(); }
 
     int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_FFMPEG; }
@@ -178,10 +178,10 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
 
         icvWriteFrame_FFMPEG_p(ffmpegWriter, (const uchar*)image.getMat().ptr(), (int)image.step(), image.cols(), image.rows(), image.channels(), 0);
     }
-    virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, bool isColor )
+    virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params )
     {
         close();
-        ffmpegWriter = icvCreateVideoWriter_FFMPEG_p( filename.c_str(), fourcc, fps, frameSize.width, frameSize.height, isColor );
+        ffmpegWriter = cvCreateVideoWriterWithParams_FFMPEG( filename.c_str(), fourcc, fps, frameSize.width, frameSize.height, params );
         return ffmpegWriter != 0;
     }
 
@@ -193,7 +193,12 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
         ffmpegWriter = 0;
     }
 
-    virtual double getProperty(int) const CV_OVERRIDE { return 0; }
+    virtual double getProperty(int propId) const CV_OVERRIDE {
+        if(!ffmpegWriter)
+            return 0;
+        return ffmpegWriter->getProperty(propId);
+    }
+
     virtual bool setProperty(int, double) CV_OVERRIDE { return false; }
     virtual bool isOpened() const CV_OVERRIDE { return ffmpegWriter != 0; }
 
@@ -207,8 +212,7 @@ cv::Ptr<cv::IVideoWriter> cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi
                                                            double fps, const cv::Size& frameSize,
                                                            const VideoWriterParameters& params)
 {
-    const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
-    cv::Ptr<CvVideoWriter_FFMPEG_proxy> writer = cv::makePtr<CvVideoWriter_FFMPEG_proxy>(filename, fourcc, fps, frameSize, isColor);
+    cv::Ptr<CvVideoWriter_FFMPEG_proxy> writer = cv::makePtr<CvVideoWriter_FFMPEG_proxy>(filename, fourcc, fps, frameSize, params);
     if (writer && writer->isOpened())
         return writer;
     return cv::Ptr<cv::IVideoWriter>();
@@ -233,7 +237,7 @@ cv::Ptr<cv::IVideoWriter> cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi
 #define CAPTURE_API_VERSION 1
 #include "plugin_capture_api.hpp"
 #define WRITER_ABI_VERSION 1
-#define WRITER_API_VERSION 0
+#define WRITER_API_VERSION 1
 #include "plugin_writer_api.hpp"
 #endif
 
@@ -400,7 +404,7 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
         Mat img;
         // TODO: avoid unnecessary copying
         if (instance->retrieveFrame(stream_idx, img))
-            return callback(stream_idx, img.data, img.step, img.cols, img.rows, img.channels(), userdata);
+            return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.channels(), userdata);
         return CV_ERROR_FAIL;
     }
     catch (const std::exception& e)
@@ -426,7 +430,7 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
         Mat img;
         // TODO: avoid unnecessary copying
         if (instance->retrieveFrame(stream_idx, img))
-            return callback(stream_idx, img.data, img.step, img.cols, img.rows, img.type(), userdata);
+            return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.type(), userdata);
         return CV_ERROR_FAIL;
     }
     catch (const std::exception& e)
@@ -443,14 +447,17 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
 #endif
 
 static
-CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
-                                    CV_OUT CvPluginWriter* handle)
+CvResult CV_API_CALL cv_writer_open_with_params(
+        const char* filename, int fourcc, double fps, int width, int height,
+        int* params, unsigned n_params,
+        CV_OUT CvPluginWriter* handle)
 {
     Size sz(width, height);
     CvVideoWriter_FFMPEG_proxy* wrt = 0;
     try
     {
-        wrt = new CvVideoWriter_FFMPEG_proxy(filename, fourcc, fps, sz, isColor != 0);
+        VideoWriterParameters parameters(params, n_params);
+        wrt = new CvVideoWriter_FFMPEG_proxy(filename, fourcc, fps, sz, parameters);
         if(wrt && wrt->isOpened())
         {
             *handle = (CvPluginWriter)wrt;
@@ -470,6 +477,14 @@ CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps
     return CV_ERROR_FAIL;
 }
 
+static
+CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
+    CV_OUT CvPluginWriter* handle)
+{
+    int params[2] = { VIDEOWRITER_PROP_IS_COLOR, isColor };
+    return cv_writer_open_with_params(filename, fourcc, fps, width, height, params, 1, handle);
+}
+
 static
 CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle)
 {
@@ -481,9 +496,22 @@ CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle)
 }
 
 static
-CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter /*handle*/, int /*prop*/, CV_OUT double* /*val*/)
+CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter handle, int prop, CV_OUT double* val)
 {
-    return CV_ERROR_FAIL;
+    if (!handle)
+        return CV_ERROR_FAIL;
+    if (!val)
+        return CV_ERROR_FAIL;
+    try
+    {
+        CvVideoWriter_FFMPEG_proxy* instance = (CvVideoWriter_FFMPEG_proxy*)handle;
+        *val = instance->getProperty(prop);
+        return CV_ERROR_OK;
+    }
+    catch (...)
+    {
+        return CV_ERROR_FAIL;
+    }
 }
 
 static
@@ -594,6 +622,9 @@ static const OpenCV_VideoIO_Writer_Plugin_API writer_plugin_api =
         /*  4*/cv_writer_get_prop,
         /*  5*/cv_writer_set_prop,
         /*  6*/cv_writer_write
+    },
+    {
+        /*  7*/cv_writer_open_with_params
     }
 };