Skip to content

Commit 7bcb51e

Browse files
Merge pull request opencv#19460 from mikhail-nikolskiy:videoio-hw
videoio: HW decode/encode in FFMPEG backend; new properties with support in FFMPEG/GST/MSMF * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * HW acceleration in FFMPEG backend * fixes on Windows, remove D3D9 * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * improve va test * Copyright * check LIBAVUTIL_BUILD >= AV_VERSION_INT(55, 78, 100) // FFMPEG 3.4+ * CAP_MSMF test on .mp4 * .mp4 in test * .avi for GStreamer test * revert changes around seek() * cv_writer_open_with_params * params.warnUnusedParameters * VideoCaptureParameters in GStreamer * open_with_params * params->getUnused * Reduce PSNR threshold 33->32 (other tests use 30) * require FFMPEG 4.0+; PSNR 30 as in other tests * GStreamer AVI-demux plugin not installed in Ubuntu test environment? * fix build on very old ffmpeg * fix build on very old ffmpeg * fix build issues * fix build issues (static_cast) * FFMPEG built on Windows without H264 encoder? * fix for write_nothing test on VAAPI * fix warnings * fix cv_writer_get_prop in plugins * use avcodec_get_hw_frames_parameters; more robust fallback to SW codecs * internal function hw_check_device() for device check/logging * two separate tests for HW read and write * image size 640x480 in encode test * WITH_VA=ON (only .h headers used in OpenCV, no linkage dependency) * exception on VP9 SW encoder? * rebase master; refine info message * videoio: fix FFmpeg standalone plugin build * videoio(ffmpeg): eliminate MSVC build warnings * address review comments * videoio(hw): update videocapture_acceleration.read test - remove parallel decoding by SW code path - check PSNR against the original generated image * videoio: minor fixes * videoio(test): disable unsupported MSMF cases (SW and HW) * videoio(test): update PSNR thresholds for HW acceleration read * videoio(test): update debug messages * "hw_acceleration" whitelisting parameter * little optimization in test * D3D11VA supports decoders, doesn't support encoders * videoio(test): adjust PSNR threshold in write_read_position tests * videoio(ffmpeg): fix rejecting on acceleration device name mismatch * videoio(ffmpeg): fix compilation USE_AV_HW_CODECS=0, add more debug logging * videoio: rework VideoAccelerationType behavior - enum is not a bitset - default value is backend specific - only '_NONE' and '_ANY' may fallback on software processing - specific H/W acceleration doesn't fallback on software processing. It fails if there is no support for specified H/W acceleration. * videoio(test): fix for current FFmpeg wrapper Co-authored-by: Alexander Alekhin <alexander.a.alekhin@gmail.com>
1 parent f70e80a commit 7bcb51e

File tree

13 files changed

+2211
-218
lines changed

13 files changed

+2211
-218
lines changed

modules/videoio/include/opencv2/videoio.hpp

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ namespace cv
7878
//! @{
7979

8080

81-
/** @brief %VideoCapture API backends identifier.
81+
/** @brief cv::VideoCapture API backends identifier.
8282
8383
Select preferred API for a capture object.
8484
To be used in the VideoCapture::VideoCapture() constructor or VideoCapture::open()
@@ -124,7 +124,7 @@ enum VideoCaptureAPIs {
124124
CAP_UEYE = 2500, //!< uEye Camera API
125125
};
126126

127-
/** @brief %VideoCapture generic properties identifier.
127+
/** @brief cv::VideoCapture generic properties identifier.
128128
129129
Reading / writing properties involves many layers. Some unexpected result might happens along this chain.
130130
Effective behaviour depends from device hardware, driver and API Backend.
@@ -182,12 +182,14 @@ enum VideoCaptureProperties {
182182
CAP_PROP_BITRATE =47, //!< (read-only) Video bitrate in kbits/s
183183
CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only)
184184
CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499)
185+
CAP_PROP_HW_ACCELERATION=50, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in cv::VideoCapture constructor / .open() method. Default value is backend-specific.
186+
CAP_PROP_HW_DEVICE =51, //!< (**open-only**) Hardware device index (select GPU if multiple available)
185187
#ifndef CV_DOXYGEN
186188
CV__CAP_PROP_LATEST
187189
#endif
188190
};
189191

190-
/** @brief %VideoWriter generic properties identifier.
192+
/** @brief cv::VideoWriter generic properties identifier.
191193
@sa VideoWriter::get(), VideoWriter::set()
192194
*/
193195
enum VideoWriterProperties {
@@ -196,14 +198,39 @@ enum VideoWriterProperties {
196198
VIDEOWRITER_PROP_NSTRIPES = 3, //!< Number of stripes for parallel encoding. -1 for auto detection.
197199
VIDEOWRITER_PROP_IS_COLOR = 4, //!< If it is not zero, the encoder will expect and encode color frames, otherwise it
198200
//!< will work with grayscale frames.
199-
VIDEOWRITER_PROP_DEPTH = 5 //!< Defaults to CV_8U.
201+
VIDEOWRITER_PROP_DEPTH = 5, //!< Defaults to CV_8U.
202+
VIDEOWRITER_PROP_HW_ACCELERATION = 6, //!< (**open-only**) Hardware acceleration type (see #VideoAccelerationType). Setting supported only via `params` parameter in VideoWriter constructor / .open() method. Default value is backend-specific.
203+
VIDEOWRITER_PROP_HW_DEVICE = 7, //!< (**open-only**) Hardware device index (select GPU if multiple available)
204+
#ifndef CV_DOXYGEN
205+
CV__VIDEOWRITER_PROP_LATEST
206+
#endif
200207
};
201208

202209
//! @} videoio_flags_base
203210

204211
//! @addtogroup videoio_flags_others
205212
//! @{
206213

214+
/** @brief Video Acceleration type
215+
*
216+
* Used as value in #CAP_PROP_HW_ACCELERATION and #VIDEOWRITER_PROP_HW_ACCELERATION
217+
*
218+
* @note In case of FFmpeg backend, it translated to enum AVHWDeviceType (https://github.com/FFmpeg/FFmpeg/blob/master/libavutil/hwcontext.h)
219+
*/
220+
enum VideoAccelerationType
221+
{
222+
VIDEO_ACCELERATION_NONE = 0, //!< Do not require any specific H/W acceleration, prefer software processing.
223+
//!< Reading of this value means that special H/W accelerated handling is not added or not detected by OpenCV.
224+
225+
VIDEO_ACCELERATION_ANY = 1, //!< Prefer to use H/W acceleration. If no one supported, then fallback to software processing.
226+
//!< @note H/W acceleration may require special configuration of used environment.
227+
//!< @note Results in encoding scenario may differ between software and hardware accelerated encoders.
228+
229+
VIDEO_ACCELERATION_D3D11 = 2, //!< DirectX 11
230+
VIDEO_ACCELERATION_VAAPI = 3, //!< VAAPI
231+
VIDEO_ACCELERATION_MFX = 4, //!< libmfx (Intel MediaSDK/oneVPL)
232+
};
233+
207234
/** @name IEEE 1394 drivers
208235
@{
209236
*/

modules/videoio/src/backend_plugin.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ class PluginCapture : public cv::IVideoCapture
415415
if (plugin_api->api_header.api_version >= 1 && plugin_api->v1.Capture_open_with_params)
416416
{
417417
std::vector<int> vint_params = params.getIntVector();
418-
int* c_params = &vint_params[0];
418+
int* c_params = vint_params.data();
419419
unsigned n_params = (unsigned)(vint_params.size() / 2);
420420

421421
if (CV_ERROR_OK == plugin_api->v1.Capture_open_with_params(
@@ -547,7 +547,7 @@ class PluginWriter : public cv::IVideoWriter
547547
}
548548
if (params.warnUnusedParameters())
549549
{
550-
CV_LOG_ERROR(NULL, "VIDEOIO/FFMPEG: unsupported parameters in VideoWriter, see logger INFO channel for details");
550+
CV_LOG_ERROR(NULL, "VIDEOIO: unsupported parameters in VideoWriter, see logger INFO channel for details");
551551
return Ptr<PluginWriter>();
552552
}
553553
if (CV_ERROR_OK == plugin_api->v0.Writer_open(filename.c_str(), fourcc, fps, sz.width, sz.height, isColor, &writer))

modules/videoio/src/backend_static.cpp

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,17 @@ void applyParametersFallback(const Ptr<IVideoCapture>& cap, const VideoCapturePa
1919
{
2020
double value = params.get<double>(prop, -1);
2121
CV_LOG_INFO(NULL, "VIDEOIO: apply parameter: [" << prop << "]=" <<
22-
cv::format("%g / %lld / 0x%16llx", value, (long long)value, (long long)value));
22+
cv::format("%g / %lld / 0x%016llx", value, (long long)value, (long long)value));
2323
if (!cap->setProperty(prop, value))
2424
{
25-
CV_Error_(cv::Error::StsNotImplemented, ("VIDEOIO: Failed to apply invalid or unsupported parameter: [%d]=%g / %lld / 0x%08llx", prop, value, (long long)value, (long long)value));
25+
if (prop != CAP_PROP_HW_ACCELERATION && prop != CAP_PROP_HW_DEVICE) { // optional parameters
26+
CV_Error_(cv::Error::StsNotImplemented, ("VIDEOIO: Failed to apply invalid or unsupported parameter: [%d]=%g / %lld / 0x%08llx", prop, value, (long long)value, (long long)value));
27+
}
2628
}
2729
}
2830
// NB: there is no dedicated "commit" parameters event, implementations should commit after each property automatically
2931
}
3032

31-
3233
// Legacy API. Modern API with parameters is below
3334
class StaticBackend: public IBackend
3435
{

modules/videoio/src/cap_ffmpeg.cpp

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -165,7 +165,7 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
165165
{
166166
public:
167167
CvVideoWriter_FFMPEG_proxy() { ffmpegWriter = 0; }
168-
CvVideoWriter_FFMPEG_proxy(const cv::String& filename, int fourcc, double fps, cv::Size frameSize, bool isColor) { ffmpegWriter = 0; open(filename, fourcc, fps, frameSize, isColor); }
168+
CvVideoWriter_FFMPEG_proxy(const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params) { ffmpegWriter = 0; open(filename, fourcc, fps, frameSize, params); }
169169
virtual ~CvVideoWriter_FFMPEG_proxy() { close(); }
170170

171171
int getCaptureDomain() const CV_OVERRIDE { return cv::CAP_FFMPEG; }
@@ -178,10 +178,10 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
178178

179179
icvWriteFrame_FFMPEG_p(ffmpegWriter, (const uchar*)image.getMat().ptr(), (int)image.step(), image.cols(), image.rows(), image.channels(), 0);
180180
}
181-
virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, bool isColor )
181+
virtual bool open( const cv::String& filename, int fourcc, double fps, cv::Size frameSize, const VideoWriterParameters& params )
182182
{
183183
close();
184-
ffmpegWriter = icvCreateVideoWriter_FFMPEG_p( filename.c_str(), fourcc, fps, frameSize.width, frameSize.height, isColor );
184+
ffmpegWriter = cvCreateVideoWriterWithParams_FFMPEG( filename.c_str(), fourcc, fps, frameSize.width, frameSize.height, params );
185185
return ffmpegWriter != 0;
186186
}
187187

@@ -193,7 +193,12 @@ class CvVideoWriter_FFMPEG_proxy CV_FINAL :
193193
ffmpegWriter = 0;
194194
}
195195

196-
virtual double getProperty(int) const CV_OVERRIDE { return 0; }
196+
virtual double getProperty(int propId) const CV_OVERRIDE {
197+
if(!ffmpegWriter)
198+
return 0;
199+
return ffmpegWriter->getProperty(propId);
200+
}
201+
197202
virtual bool setProperty(int, double) CV_OVERRIDE { return false; }
198203
virtual bool isOpened() const CV_OVERRIDE { return ffmpegWriter != 0; }
199204

@@ -207,8 +212,7 @@ cv::Ptr<cv::IVideoWriter> cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi
207212
double fps, const cv::Size& frameSize,
208213
const VideoWriterParameters& params)
209214
{
210-
const bool isColor = params.get(VIDEOWRITER_PROP_IS_COLOR, true);
211-
cv::Ptr<CvVideoWriter_FFMPEG_proxy> writer = cv::makePtr<CvVideoWriter_FFMPEG_proxy>(filename, fourcc, fps, frameSize, isColor);
215+
cv::Ptr<CvVideoWriter_FFMPEG_proxy> writer = cv::makePtr<CvVideoWriter_FFMPEG_proxy>(filename, fourcc, fps, frameSize, params);
212216
if (writer && writer->isOpened())
213217
return writer;
214218
return cv::Ptr<cv::IVideoWriter>();
@@ -233,7 +237,7 @@ cv::Ptr<cv::IVideoWriter> cvCreateVideoWriter_FFMPEG_proxy(const std::string& fi
233237
#define CAPTURE_API_VERSION 1
234238
#include "plugin_capture_api.hpp"
235239
#define WRITER_ABI_VERSION 1
236-
#define WRITER_API_VERSION 0
240+
#define WRITER_API_VERSION 1
237241
#include "plugin_writer_api.hpp"
238242
#endif
239243

@@ -400,7 +404,7 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
400404
Mat img;
401405
// TODO: avoid unnecessary copying
402406
if (instance->retrieveFrame(stream_idx, img))
403-
return callback(stream_idx, img.data, img.step, img.cols, img.rows, img.channels(), userdata);
407+
return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.channels(), userdata);
404408
return CV_ERROR_FAIL;
405409
}
406410
catch (const std::exception& e)
@@ -426,7 +430,7 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
426430
Mat img;
427431
// TODO: avoid unnecessary copying
428432
if (instance->retrieveFrame(stream_idx, img))
429-
return callback(stream_idx, img.data, img.step, img.cols, img.rows, img.type(), userdata);
433+
return callback(stream_idx, img.data, (int)img.step, img.cols, img.rows, img.type(), userdata);
430434
return CV_ERROR_FAIL;
431435
}
432436
catch (const std::exception& e)
@@ -443,14 +447,17 @@ CvResult CV_API_CALL cv_capture_retrieve(CvPluginCapture handle, int stream_idx,
443447
#endif
444448

445449
static
446-
CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
447-
CV_OUT CvPluginWriter* handle)
450+
CvResult CV_API_CALL cv_writer_open_with_params(
451+
const char* filename, int fourcc, double fps, int width, int height,
452+
int* params, unsigned n_params,
453+
CV_OUT CvPluginWriter* handle)
448454
{
449455
Size sz(width, height);
450456
CvVideoWriter_FFMPEG_proxy* wrt = 0;
451457
try
452458
{
453-
wrt = new CvVideoWriter_FFMPEG_proxy(filename, fourcc, fps, sz, isColor != 0);
459+
VideoWriterParameters parameters(params, n_params);
460+
wrt = new CvVideoWriter_FFMPEG_proxy(filename, fourcc, fps, sz, parameters);
454461
if(wrt && wrt->isOpened())
455462
{
456463
*handle = (CvPluginWriter)wrt;
@@ -470,6 +477,14 @@ CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps
470477
return CV_ERROR_FAIL;
471478
}
472479

480+
static
481+
CvResult CV_API_CALL cv_writer_open(const char* filename, int fourcc, double fps, int width, int height, int isColor,
482+
CV_OUT CvPluginWriter* handle)
483+
{
484+
int params[2] = { VIDEOWRITER_PROP_IS_COLOR, isColor };
485+
return cv_writer_open_with_params(filename, fourcc, fps, width, height, params, 1, handle);
486+
}
487+
473488
static
474489
CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle)
475490
{
@@ -481,9 +496,22 @@ CvResult CV_API_CALL cv_writer_release(CvPluginWriter handle)
481496
}
482497

483498
static
484-
CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter /*handle*/, int /*prop*/, CV_OUT double* /*val*/)
499+
CvResult CV_API_CALL cv_writer_get_prop(CvPluginWriter handle, int prop, CV_OUT double* val)
485500
{
486-
return CV_ERROR_FAIL;
501+
if (!handle)
502+
return CV_ERROR_FAIL;
503+
if (!val)
504+
return CV_ERROR_FAIL;
505+
try
506+
{
507+
CvVideoWriter_FFMPEG_proxy* instance = (CvVideoWriter_FFMPEG_proxy*)handle;
508+
*val = instance->getProperty(prop);
509+
return CV_ERROR_OK;
510+
}
511+
catch (...)
512+
{
513+
return CV_ERROR_FAIL;
514+
}
487515
}
488516

489517
static
@@ -594,6 +622,9 @@ static const OpenCV_VideoIO_Writer_Plugin_API writer_plugin_api =
594622
/* 4*/cv_writer_get_prop,
595623
/* 5*/cv_writer_set_prop,
596624
/* 6*/cv_writer_write
625+
},
626+
{
627+
/* 7*/cv_writer_open_with_params
597628
}
598629
};
599630

0 commit comments

Comments
 (0)