Skip to content

Commit 7ac648c

Browse files
tsenstalalek
authored andcommitted
Merge pull request #2097 from tsenst:robust_optical_flow_fix
optflow: RLOF fixes (python-binding, status flag, flat textures) (#2097) * * Bugfix for python-binding related to issue #2094. Copying the prevPtsMat to prevPoints was broken, when using python-binding. * * Connect the blurred image correctly to CROSS support region segmentation. This allows to compute more robust support region shapes. Commit refers to issue #2094. * * Bugfix to avoid the unintended assertion on flat textures discussed in issue #2094. * * Bugfix on SparseRLOFOpticalFlow status was wrongly to to 0 if forward backward error < threshold, which is wrong. Inequation has been corrected. Refers to in issue #2087. * small documentation changes * bugfix on assertions and small test changes
1 parent 644f428 commit 7ac648c

File tree

6 files changed

+38
-21
lines changed

6 files changed

+38
-21
lines changed

modules/optflow/include/opencv2/optflow/rlofflow.hpp

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@ enum InterpolationType
3535

3636
/** @brief This is used store and set up the parameters of the robust local optical flow (RLOF) algoritm.
3737
*
38-
* The RLOF is a fast local optical flow approach similar to the pyramidal iterative Lucas-Kanade method as
38+
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
39+
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
3940
* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
4041
* This RLOF implementation can be seen as an improved pyramidal iterative Lucas-Kanade and includes
4142
* a set of improving modules. The main improvements in respect to the pyramidal iterative Lucas-Kanade
@@ -197,7 +198,8 @@ class CV_EXPORTS_W RLOFOpticalFlowParameter{
197198
/** @brief Fast dense optical flow computation based on robust local optical flow (RLOF) algorithms and sparse-to-dense interpolation
198199
* scheme.
199200
*
200-
* The RLOF is a fast local optical flow approach similar to the pyramidal iterative Lucas-Kanade method as
201+
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
202+
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
201203
* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
202204
*
203205
* The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).
@@ -350,7 +352,8 @@ class CV_EXPORTS_W DenseRLOFOpticalFlow : public DenseOpticalFlow
350352

351353
/** @brief Class used for calculation sparse optical flow and feature tracking with robust local optical flow (RLOF) algorithms.
352354
*
353-
* The RLOF is a fast local optical flow approach similar to the pyramidal iterative Lucas-Kanade method as
355+
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
356+
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
354357
* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
355358
*
356359
* For the RLOF configuration see optflow::RLOFOpticalFlowParameter for further details.
@@ -396,7 +399,8 @@ class CV_EXPORTS_W SparseRLOFOpticalFlow : public SparseOpticalFlow
396399

397400
/** @brief Fast dense optical flow computation based on robust local optical flow (RLOF) algorithms and sparse-to-dense interpolation scheme.
398401
*
399-
* The RLOF is a fast local optical flow approach similar to the pyramidal iterative Lucas-Kanade method as
402+
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
403+
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
400404
* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
401405
*
402406
* The sparse-to-dense interpolation scheme allows for fast computation of dense optical flow using RLOF (see @cite Geistert2016).
@@ -452,7 +456,8 @@ CV_EXPORTS_W void calcOpticalFlowDenseRLOF(InputArray I0, InputArray I1, InputOu
452456
/** @brief Calculates fast optical flow for a sparse feature set using the robust local optical flow (RLOF) similar
453457
* to optflow::calcOpticalFlowPyrLK().
454458
*
455-
* The RLOF is a fast local optical flow approach similar to the pyramidal iterative Lucas-Kanade method as
459+
* The RLOF is a fast local optical flow approach described in @cite Senst2012 @cite Senst2013 @cite Senst2014
460+
* and @cite Senst2016 similar to the pyramidal iterative Lucas-Kanade method as
456461
* proposed by @cite Bouguet00. The implementation is derived from optflow::calcOpticalFlowPyrLK().
457462
*
458463
* @param prevImg first 8-bit input image. If The cross-based RLOF is used (by selecting optflow::RLOFOpticalFlowParameter::supportRegionType

modules/optflow/src/rlof/rlof_invokerbase.hpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,13 +102,13 @@ void getLocalPatch(
102102
bounds = src.at<cv::Vec4i>(_r,c);
103103
roi.x = bounds.val[0] - border_left;
104104
roi.width = bounds.val[1] - bounds.val[0];
105+
cv::Mat(winPointMask, roi).setTo(1);
105106
}
106107
else
107108
{
108109
bounds.val[0] = border_left;
109110
bounds.val[1] = border_left + roi.width;
110111
}
111-
cv::Mat(winPointMask, roi).setTo(1);
112112
min_c = MIN(min_c, bounds.val[0]);
113113
max_c = MAX(max_c, bounds.val[1]);
114114
noPoints += roi.width;

modules/optflow/src/rlof/rlof_localflow.cpp

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -354,11 +354,15 @@ int buildOpticalFlowPyramidScale(InputArray _img, OutputArrayOfArrays pyramid, S
354354

355355
return maxLevel;
356356
}
357-
int CImageBuffer::buildPyramid(cv::Size winSize, int maxLevel, float levelScale[2])
357+
358+
int CImageBuffer::buildPyramid(cv::Size winSize, int maxLevel, float levelScale[2],bool withBlurredImage )
358359
{
359-
if (m_Overwrite == false)
360+
if (! m_Overwrite)
360361
return m_maxLevel;
361-
m_maxLevel = buildOpticalFlowPyramidScale(m_Image, m_ImagePyramid, winSize, maxLevel, false, 4, 0, true, levelScale);
362+
if (withBlurredImage)
363+
m_maxLevel = buildOpticalFlowPyramidScale(m_BlurredImage, m_ImagePyramid, winSize, maxLevel, false, 4, 0, true, levelScale);
364+
else
365+
m_maxLevel = buildOpticalFlowPyramidScale(m_Image, m_ImagePyramid, winSize, maxLevel, false, 4, 0, true, levelScale);
362366
return m_maxLevel;
363367
}
364368

@@ -407,12 +411,12 @@ void calcLocalOpticalFlowCore(
407411
float levelScale[2] = { 2.f,2.f };
408412

409413
int maxLevel = prevPyramids[0]->buildPyramid(cv::Size(iWinSize, iWinSize), param.maxLevel, levelScale);
410-
411414
maxLevel = currPyramids[0]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale);
415+
412416
if (useAdditionalRGB)
413417
{
414-
prevPyramids[1]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale);
415-
currPyramids[1]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale);
418+
prevPyramids[1]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale, true);
419+
currPyramids[1]->buildPyramid(cv::Size(iWinSize, iWinSize), maxLevel, levelScale, true);
416420
}
417421

418422
if ((criteria.type & TermCriteria::COUNT) == 0)
@@ -661,7 +665,8 @@ void calcLocalOpticalFlow(
661665
prevPyramids[0]->m_Overwrite = true;
662666
currPyramids[0]->m_Overwrite = true;
663667
prevPyramids[1]->m_Overwrite = true;
664-
currPyramids[1]->m_Overwrite = true;
668+
// perform blurring and build blur pyramid only for the prev image
669+
currPyramids[1]->m_Overwrite = false;
665670
if (prevImage.type() == CV_8UC3)
666671
{
667672
prevPyramids[0]->setGrayFromRGB(prevImage);

modules/optflow/src/rlof/rlof_localflow.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ class CImageBuffer
9393
cv::GaussianBlur(inp, m_BlurredImage, cv::Size(7,7), -1);
9494
}
9595

96-
int buildPyramid(cv::Size winSize, int maxLevel, float levelScale[2]);
96+
int buildPyramid(cv::Size winSize, int maxLevel, float levelScale[2], bool withBlurredImage = false);
9797
cv::Mat & getImage(int level) {return m_ImagePyramid[level];}
9898

9999
std::vector<cv::Mat> m_ImagePyramid;

modules/optflow/src/rlofflow.cpp

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -303,22 +303,29 @@ class SparseRLOFOpticalFlowImpl : public SparseRLOFOpticalFlow
303303

304304
int npoints = 0;
305305
CV_Assert((npoints = prevPtsMat.checkVector(2, CV_32F, true)) >= 0);
306-
307306
if (npoints == 0)
308307
{
309308
nextPts.release();
310309
status.release();
311310
err.release();
312311
return;
313312
}
314-
315313
Mat nextPtsMat = nextPts.getMat();
316314
CV_Assert(nextPtsMat.checkVector(2, CV_32F, true) == npoints);
315+
317316
std::vector<cv::Point2f> prevPoints(npoints), nextPoints(npoints), refPoints;
318-
prevPtsMat.copyTo(cv::Mat(1, npoints, CV_32FC2, &prevPoints[0]));
319-
if (param->useInitialFlow )
320-
nextPtsMat.copyTo(cv::Mat(1, nextPtsMat.cols, CV_32FC2, &nextPoints[0]));
321317

318+
if (prevPtsMat.channels() != 2)
319+
prevPtsMat = prevPtsMat.reshape(2, npoints);
320+
321+
prevPtsMat.copyTo(prevPoints);
322+
323+
if (param->useInitialFlow )
324+
{
325+
if (nextPtsMat.channels() != 2)
326+
nextPtsMat = nextPtsMat.reshape(2, npoints);
327+
nextPtsMat.copyTo(nextPoints);
328+
}
322329
cv::Mat statusMat;
323330
cv::Mat errorMat;
324331
if (status.needed() || forwardBackwardThreshold > 0)
@@ -346,7 +353,7 @@ class SparseRLOFOpticalFlowImpl : public SparseRLOFOpticalFlow
346353
{
347354
Point2f diff = refPoints[r] - prevPoints[r];
348355
errorMat.at<float>(r) = sqrt(diff.x * diff.x + diff.y * diff.y);
349-
if (errorMat.at<float>(r) <= forwardBackwardThreshold)
356+
if (errorMat.at<float>(r) > forwardBackwardThreshold)
350357
statusMat.at<uchar>(r) = 0;
351358
}
352359

modules/optflow/test/test_OF_accuracy.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,7 @@ TEST(DenseOpticalFlow_RLOF, ReferenceAccuracy)
260260

261261
ASSERT_EQ(GT.rows, flow.rows);
262262
ASSERT_EQ(GT.cols, flow.cols);
263-
EXPECT_LE(calcRMSE(GT, flow), 0.44f);
263+
EXPECT_LE(calcRMSE(GT, flow), 0.46f);
264264

265265
algo->setInterpolation(INTERP_GEO);
266266
algo->calc(frame1, frame2, flow);

0 commit comments

Comments
 (0)