Autorotation for mp4 streams with metadata

- Add VideoCapture camera orientation property for mp4 videos with camera orientation meta.
- Add auto rotation for 90, 180, 270 degrees using cv::rotate

Author: Alexey Lapshin

modules/videoio/include/opencv2/videoio.hpp

@@ -177,6 +177,8 @@ enum VideoCaptureProperties {
        CAP_PROP_WB_TEMPERATURE=45, //!< white-balance color temperature
        CAP_PROP_CODEC_PIXEL_FORMAT =46,    //!< (read-only) codec's pixel format. 4-character code - see VideoWriter::fourcc . Subset of [AV_PIX_FMT_*](https://github.com/FFmpeg/FFmpeg/blob/master/libavcodec/raw.c) or -1 if unknown
        CAP_PROP_BITRATE       =47, //!< (read-only) Video bitrate in kbits/s
+       CAP_PROP_ORIENTATION_META=48, //!< (read-only) Frame rotation defined by stream meta (applicable for FFmpeg back-end only)
+       CAP_PROP_ORIENTATION_AUTO=49, //!< if true - rotates output frames of CvCapture considering video file's metadata  (applicable for FFmpeg back-end only) (https://github.com/opencv/opencv/issues/15499)
 #ifndef CV_DOXYGEN
        CV__CAP_PROP_LATEST
 #endif

modules/videoio/src/cap_ffmpeg.cpp

@@ -229,13 +229,12 @@ class CvCapture_FFMPEG_proxy CV_FINAL : public cv::IVideoCapture
     }
     virtual bool retrieveFrame(int, cv::OutputArray frame) CV_OVERRIDE
     {
-        unsigned char* data = 0;
-        int step=0, width=0, height=0, cn=0;
-
+        cv::Mat mat;
         if (!ffmpegCapture ||
-           !icvRetrieveFrame_FFMPEG_p(ffmpegCapture, &data, &step, &width, &height, &cn))
+           !icvRetrieveFrame_FFMPEG_p(ffmpegCapture, mat))
             return false;
-        cv::Mat(height, width, CV_MAKETYPE(CV_8U, cn), data, step).copyTo(frame);
+
+        mat.copyTo(frame);
         return true;
     }
     virtual bool open( const cv::String& filename )
@@ -262,6 +261,8 @@ class CvCapture_FFMPEG_proxy CV_FINAL : public cv::IVideoCapture
 
 protected:
     CvCapture_FFMPEG* ffmpegCapture;
+private:
+
 };
 
 } // namespace

modules/videoio/src/cap_ffmpeg_api.hpp

@@ -28,7 +28,9 @@ enum
     CV_FFMPEG_CAP_PROP_SAR_NUM=40,
     CV_FFMPEG_CAP_PROP_SAR_DEN=41,
     CV_FFMPEG_CAP_PROP_CODEC_PIXEL_FORMAT=46,
-    CV_FFMPEG_CAP_PROP_BITRATE=47
+    CV_FFMPEG_CAP_PROP_BITRATE=47,
+    CV_FFMPEG_CAP_PROP_ORIENTATION_META=48,
+    CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO=49
 };
 
 typedef struct CvCapture_FFMPEG CvCapture_FFMPEG;
@@ -39,8 +41,7 @@ OPENCV_FFMPEG_API int cvSetCaptureProperty_FFMPEG(struct CvCapture_FFMPEG* cap,
                                                   int prop, double value);
 OPENCV_FFMPEG_API double cvGetCaptureProperty_FFMPEG(struct CvCapture_FFMPEG* cap, int prop);
 OPENCV_FFMPEG_API int cvGrabFrame_FFMPEG(struct CvCapture_FFMPEG* cap);
-OPENCV_FFMPEG_API int cvRetrieveFrame_FFMPEG(struct CvCapture_FFMPEG* capture, unsigned char** data,
-                                             int* step, int* width, int* height, int* cn);
+OPENCV_FFMPEG_API int cvRetrieveFrame_FFMPEG(struct CvCapture_FFMPEG* capture, cv::Mat &mat);
 OPENCV_FFMPEG_API void cvReleaseCapture_FFMPEG(struct CvCapture_FFMPEG** cap);
 
 OPENCV_FFMPEG_API struct CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG(const char* filename,
@@ -52,8 +53,7 @@ OPENCV_FFMPEG_API void cvReleaseVideoWriter_FFMPEG(struct CvVideoWriter_FFMPEG**
 typedef CvCapture_FFMPEG* (*CvCreateFileCapture_Plugin)( const char* filename );
 typedef CvCapture_FFMPEG* (*CvCreateCameraCapture_Plugin)( int index );
 typedef int (*CvGrabFrame_Plugin)( CvCapture_FFMPEG* capture_handle );
-typedef int (*CvRetrieveFrame_Plugin)( CvCapture_FFMPEG* capture_handle, unsigned char** data, int* step,
-                                       int* width, int* height, int* cn );
+typedef int (*CvRetrieveFrame_Plugin)( CvCapture_FFMPEG* capture_handle, cv::Mat &mat);
 typedef int (*CvSetCaptureProperty_Plugin)( CvCapture_FFMPEG* capture_handle, int prop_id, double value );
 typedef double (*CvGetCaptureProperty_Plugin)( CvCapture_FFMPEG* capture_handle, int prop_id );
 typedef void (*CvReleaseCapture_Plugin)( CvCapture_FFMPEG** capture_handle );

modules/videoio/src/cap_ffmpeg_impl.hpp

@@ -481,7 +481,8 @@ struct CvCapture_FFMPEG
     double getProperty(int) const;
     bool setProperty(int, double);
     bool grabFrame();
-    bool retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn);
+    bool retrieveFrame(int, cv::Mat &mat);
+    void rotateFrame(cv::Mat &mat) const;
 
     void init();
 
@@ -497,6 +498,7 @@ struct CvCapture_FFMPEG
     double  r2d(AVRational r) const;
     int64_t dts_to_frame_number(int64_t dts);
     double  dts_to_sec(int64_t dts) const;
+    void    get_rotation_angle();
 
     AVFormatContext * ic;
     AVCodec         * avcodec;
@@ -512,6 +514,8 @@ struct CvCapture_FFMPEG
 
     int64_t frame_number, first_frame_number;
 
+    bool   rotation_auto;
+    int    rotation_angle; // valid 0, 90, 180, 270
     double eps_zero;
 /*
    'filename' contains the filename of the videosource,
@@ -560,6 +564,9 @@ void CvCapture_FFMPEG::init()
     frame_number = 0;
     eps_zero = 0.000025;
 
+    rotation_auto = true;
+    rotation_angle = 0;
+
 #if LIBAVFORMAT_BUILD >= CALC_FFMPEG_VERSION(52, 111, 0)
     dict = NULL;
 #endif
@@ -1032,6 +1039,7 @@ bool CvCapture_FFMPEG::open( const char* _filename )
             frame.cn = 3;
             frame.step = 0;
             frame.data = NULL;
+            get_rotation_angle();
             break;
         }
     }
@@ -1279,21 +1287,19 @@ bool CvCapture_FFMPEG::grabFrame()
     return valid;
 }
 
-
-bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int* width, int* height, int* cn)
+bool CvCapture_FFMPEG::retrieveFrame(int, cv::Mat &mat)
 {
     if (!video_st)
         return false;
 
     if (rawMode)
     {
         AVPacket& p = bsfc ? packet_filtered : packet;
-        *data = p.data;
-        *step = p.size;
-        *width = p.size;
-        *height = 1;
-        *cn = 1;
-        return p.data != NULL;
+        if (p.data == NULL)
+            return false;
+
+        mat = cv::Mat(1, p.size, CV_MAKETYPE(CV_8U, 1), p.data, p.size);
+        return true;
     }
 
     if (!picture->data[0])
@@ -1356,15 +1362,29 @@ bool CvCapture_FFMPEG::retrieveFrame(int, unsigned char** data, int* step, int*
             rgb_picture.linesize
             );
 
-    *data = frame.data;
-    *step = frame.step;
-    *width = frame.width;
-    *height = frame.height;
-    *cn = frame.cn;
-
+    mat = cv::Mat(frame.height, frame.width, CV_MAKETYPE(CV_8U, frame.cn), frame.data, frame.step);
+    rotateFrame(mat);
     return true;
 }
 
+void CvCapture_FFMPEG::rotateFrame(cv::Mat &mat) const {
+    if(!rotation_auto || rotation_angle%360 == 0) {
+        return;
+    }
+
+    cv::RotateFlags flag;
+    if(rotation_angle == 90 || rotation_angle == -270) { // Rotate clockwise 90 degrees
+        flag = cv::ROTATE_90_CLOCKWISE;
+    } else if(rotation_angle == 270 || rotation_angle == -90) { // Rotate clockwise 270 degrees
+        flag = cv::ROTATE_90_COUNTERCLOCKWISE;
+    } else if(rotation_angle == 180 || rotation_angle == -180) { // Rotate clockwise 180 degrees
+        flag = cv::ROTATE_180;
+    } else { // Unsupported rotation
+        return;
+    }
+
+    cv::rotate(mat, mat, flag);
+}
 
 double CvCapture_FFMPEG::getProperty( int property_id ) const
 {
@@ -1389,9 +1409,9 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const
     case CV_FFMPEG_CAP_PROP_FRAME_COUNT:
         return (double)get_total_frames();
     case CV_FFMPEG_CAP_PROP_FRAME_WIDTH:
-        return (double)frame.width;
+        return (double)((rotation_auto && rotation_angle%360) ? frame.height : frame.width);
     case CV_FFMPEG_CAP_PROP_FRAME_HEIGHT:
-        return (double)frame.height;
+        return (double)((rotation_auto && rotation_angle%360) ? frame.width : frame.height);
     case CV_FFMPEG_CAP_PROP_FPS:
         return get_fps();
     case CV_FFMPEG_CAP_PROP_FOURCC:
@@ -1435,6 +1455,10 @@ double CvCapture_FFMPEG::getProperty( int property_id ) const
         break;
     case CV_FFMPEG_CAP_PROP_BITRATE:
         return static_cast<double>(get_bitrate());
+    case CV_FFMPEG_CAP_PROP_ORIENTATION_META:
+        return static_cast<double>(rotation_angle);
+    case CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO:
+        return static_cast<double>(rotation_auto);
     default:
         break;
     }
@@ -1513,6 +1537,14 @@ double CvCapture_FFMPEG::dts_to_sec(int64_t dts) const
         r2d(ic->streams[video_stream]->time_base);
 }
 
+void CvCapture_FFMPEG::get_rotation_angle()
+{
+    rotation_angle = 0;
+    AVDictionaryEntry *rotate_tag = av_dict_get(video_st->metadata, "rotate", NULL, 0);
+    if (rotate_tag != NULL)
+        rotation_angle = atoi(rotate_tag->value);
+}
+
 void CvCapture_FFMPEG::seek(int64_t _frame_number)
 {
     _frame_number = std::min(_frame_number, get_total_frames());
@@ -1608,6 +1640,9 @@ bool CvCapture_FFMPEG::setProperty( int property_id, double value )
         if (value == -1)
             return setRaw();
         return false;
+    case CV_FFMPEG_CAP_PROP_ORIENTATION_AUTO:
+        rotation_auto = static_cast<bool>(value);
+        break;
     default:
         return false;
     }
@@ -2635,9 +2670,9 @@ int cvGrabFrame_FFMPEG(CvCapture_FFMPEG* capture)
     return capture->grabFrame();
 }
 
-int cvRetrieveFrame_FFMPEG(CvCapture_FFMPEG* capture, unsigned char** data, int* step, int* width, int* height, int* cn)
+int cvRetrieveFrame_FFMPEG(CvCapture_FFMPEG* capture, cv::Mat &mat)
 {
-    return capture->retrieveFrame(0, data, step, width, height, cn);
+    return capture->retrieveFrame(0, mat);
 }
 
 CvVideoWriter_FFMPEG* cvCreateVideoWriter_FFMPEG( const char* filename, int fourcc, double fps,