From 6d30fce53bf1de47503aa800e946ac4123466a27 Mon Sep 17 00:00:00 2001 From: binbin-xu Date: Sun, 18 Jun 2017 19:25:38 +0900 Subject: [PATCH 01/17] First PR. Added the interface headers, readme, bib files of the obstruction free method. --- modules/xphoto/README.md | 1 + modules/xphoto/doc/xphoto.bib | 9 +++ modules/xphoto/include/opencv2/xphoto.hpp | 1 + .../opencv2/xphoto/obstruction_free.hpp | 77 +++++++++++++++++++ modules/xphoto/src/obstruction_free.cpp | 67 ++++++++++++++++ 5 files changed, 155 insertions(+) create mode 100644 modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp create mode 100644 modules/xphoto/src/obstruction_free.cpp diff --git a/modules/xphoto/README.md b/modules/xphoto/README.md index fc4aa8857b9..217ee269332 100644 --- a/modules/xphoto/README.md +++ b/modules/xphoto/README.md @@ -4,4 +4,5 @@ Additional photo processing algorithms 1. Color balance 2. Denoising 3. Inpainting +4. Obstruction-free photography (occlusion and reflection) diff --git a/modules/xphoto/doc/xphoto.bib b/modules/xphoto/doc/xphoto.bib index b57471ceb64..9d6ce705c8e 100644 --- a/modules/xphoto/doc/xphoto.bib +++ b/modules/xphoto/doc/xphoto.bib @@ -14,3 +14,12 @@ @inproceedings{Cheng2015 pages={1000--1008}, year={2015} } + +@article{Xue2015ObstructionFree, + author = {Tianfan Xue and Michael Rubinstein and Ce Liu and William T. Freeman}, + title = {A Computational Approach for Obstruction-Free Photography}, + journal = {ACM Transactions on Graphics (Proc. SIGGRAPH)}, + year = {2015}, + volume = {34}, + number = {4}, +} diff --git a/modules/xphoto/include/opencv2/xphoto.hpp b/modules/xphoto/include/opencv2/xphoto.hpp index 73b1e0bb3c4..70b35a1fd9c 100644 --- a/modules/xphoto/include/opencv2/xphoto.hpp +++ b/modules/xphoto/include/opencv2/xphoto.hpp @@ -50,4 +50,5 @@ #include "xphoto/white_balance.hpp" #include "xphoto/dct_image_denoising.hpp" #include "xphoto/bm3d_image_denoising.hpp" +#include "xphoto/obstruction_free.hpp" #endif diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp new file mode 100644 index 00000000000..07db11c41a9 --- /dev/null +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -0,0 +1,77 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// Redistribution and use in source and binary forms, with or without modification, +// are permitted provided that the following conditions are met: +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of the copyright holders may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_OBSTRUCTION_FREE_HPP__ +#define __OPENCV_OBSTRUCTION_FREE_HPP__ + +/** @file +@date June 18, 2017 +@author Binbin Xu +*/ + +#include + +namespace cv +{ +namespace xphoto +{ + +//! @addtogroup xphoto +//! @{ + + + /** @brief The function implements a general obstruction free approach that can remove occlusions and reflections from input image sequences without manual masks. + + See the original paper @cite Xue2015ObstructionFree for more details. + + @param src source image sequences, involving translation motions. + @param dst Obstruction-removed destination image, corresponding to the reference image, with the same size and type. In general, the reference image is the center frame of the input image. + @param mask mask (CV_8UC1), where zero pixels indicate area to be estimated to be occlusions. + */ + CV_EXPORTS void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask); + +//! @} + +} +} + +#endif // __OPENCV_INPAINTING_HPP__ diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp new file mode 100644 index 00000000000..8bb98b86237 --- /dev/null +++ b/modules/xphoto/src/obstruction_free.cpp @@ -0,0 +1,67 @@ +/*M/////////////////////////////////////////////////////////////////////////////////////// +// +// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. +// +// By downloading, copying, installing or using the software you agree to this license. +// If you do not agree to this license, do not download, install, +// copy or use the software. +// +// +// License Agreement +// For Open Source Computer Vision Library +// +// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. +// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. +// Third party copyrights are property of their respective owners. +// +// * Redistribution's of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistribution's in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// * The name of Intel Corporation may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// This software is provided by the copyright holders and contributors "as is" and +// any express or implied warranties, including, but not limited to, the implied +// warranties of merchantability and fitness for a particular purpose are disclaimed. +// In no event shall the Intel Corporation or contributors be liable for any direct, +// indirect, incidental, special, exemplary, or consequential damages +// (including, but not limited to, procurement of substitute goods or services; +// loss of use, data, or profits; or business interruption) however caused +// and on any theory of liability, whether in contract, strict liability, +// or tort (including negligence or otherwise) arising in any way out of +// the use of this software, even if advised of the possibility of such damage. +// +//M*/ + +#ifndef __OPENCV_OBSTRUCTION_FREE_CPP__ +#define __OPENCV_OBSTRUCTION_FREE_CPP__ +#ifdef __cplusplus + +#include +#include + +#include + +#include +#include +#include + +namespace cv +{ +namespace xphoto +{ +void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ + int frameNumber = srcImgs.size(); + int reference_number = (frameNumber-1)/2;; + dst.create( srcImgs[reference_number].size(), srcImgs[reference_number].type() ); + mask = Mat::zeros(dst.rows,dst.cols,CV_8UC1); + +} +} +} +#endif // __OPENCV_PCALIB_CPP__ +#endif // cplusplus From 0486e044b40676412602e953c5fce6215b13fa71 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Sun, 18 Jun 2017 21:19:27 +0900 Subject: [PATCH 02/17] Fix CmakeLists --- modules/xphoto/CMakeLists.txt | 2 +- modules/xphoto/src/obstruction_free.cpp | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/modules/xphoto/CMakeLists.txt b/modules/xphoto/CMakeLists.txt index 3e550a3c892..7c95f0841c4 100644 --- a/modules/xphoto/CMakeLists.txt +++ b/modules/xphoto/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Addon to basic photo module") -ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python) +ocv_define_module(xphoto opencv_core opencv_imgproc opencv_highgui WRAP python) diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 8bb98b86237..9c62179a308 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -46,7 +46,7 @@ #include -#include +#include #include #include @@ -60,6 +60,19 @@ void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ dst.create( srcImgs[reference_number].size(), srcImgs[reference_number].type() ); mask = Mat::zeros(dst.rows,dst.cols,CV_8UC1); + + /////////construct image pyramids////// + int pyramid_level=3; + std::vector video_coarseLeve; + for (int frame_i=0; frame_i Date: Sun, 18 Jun 2017 22:18:28 +0900 Subject: [PATCH 03/17] fix the conversion between size_t and int on the Win64 system --- modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp | 2 +- modules/xphoto/src/obstruction_free.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 07db11c41a9..cac9aa2b7da 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -63,7 +63,7 @@ namespace xphoto See the original paper @cite Xue2015ObstructionFree for more details. - @param src source image sequences, involving translation motions. + @param srcImgs source image sequences, involving translation motions. @param dst Obstruction-removed destination image, corresponding to the reference image, with the same size and type. In general, the reference image is the center frame of the input image. @param mask mask (CV_8UC1), where zero pixels indicate area to be estimated to be occlusions. */ diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 9c62179a308..9b4365e9411 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -55,8 +55,8 @@ namespace cv namespace xphoto { void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ - int frameNumber = srcImgs.size(); - int reference_number = (frameNumber-1)/2;; + size_t frameNumber = srcImgs.size(); + size_t reference_number = (frameNumber-1)/2;; dst.create( srcImgs[reference_number].size(), srcImgs[reference_number].type() ); mask = Mat::zeros(dst.rows,dst.cols,CV_8UC1); @@ -64,7 +64,7 @@ void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ /////////construct image pyramids////// int pyramid_level=3; std::vector video_coarseLeve; - for (int frame_i=0; frame_i Date: Thu, 22 Jun 2017 01:21:00 +0900 Subject: [PATCH 04/17] fix the problems commented by the reviewer sovrasov --- modules/xphoto/CMakeLists.txt | 2 +- .../opencv2/xphoto/obstruction_free.hpp | 81 ++++++++--------- modules/xphoto/src/obstruction_free.cpp | 90 ++++++++++--------- 3 files changed, 88 insertions(+), 85 deletions(-) diff --git a/modules/xphoto/CMakeLists.txt b/modules/xphoto/CMakeLists.txt index 7c95f0841c4..3e550a3c892 100644 --- a/modules/xphoto/CMakeLists.txt +++ b/modules/xphoto/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Addon to basic photo module") -ocv_define_module(xphoto opencv_core opencv_imgproc opencv_highgui WRAP python) +ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index cac9aa2b7da..55c08f215bb 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -1,44 +1,45 @@ /*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// Redistribution and use in source and binary forms, with or without modification, -// are permitted provided that the following conditions are met: -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of the copyright holders may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // (3-clause BSD License) + // + // Copyright (C) 2015, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + #ifndef __OPENCV_OBSTRUCTION_FREE_HPP__ #define __OPENCV_OBSTRUCTION_FREE_HPP__ diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 9b4365e9411..b5403cb47a5 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -1,45 +1,49 @@ /*M/////////////////////////////////////////////////////////////////////////////////////// -// -// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. -// -// By downloading, copying, installing or using the software you agree to this license. -// If you do not agree to this license, do not download, install, -// copy or use the software. -// -// -// License Agreement -// For Open Source Computer Vision Library -// -// Copyright (C) 2000-2008, Intel Corporation, all rights reserved. -// Copyright (C) 2009-2011, Willow Garage Inc., all rights reserved. -// Third party copyrights are property of their respective owners. -// -// * Redistribution's of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistribution's in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// * The name of Intel Corporation may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// This software is provided by the copyright holders and contributors "as is" and -// any express or implied warranties, including, but not limited to, the implied -// warranties of merchantability and fitness for a particular purpose are disclaimed. -// In no event shall the Intel Corporation or contributors be liable for any direct, -// indirect, incidental, special, exemplary, or consequential damages -// (including, but not limited to, procurement of substitute goods or services; -// loss of use, data, or profits; or business interruption) however caused -// and on any theory of liability, whether in contract, strict liability, -// or tort (including negligence or otherwise) arising in any way out of -// the use of this software, even if advised of the possibility of such damage. -// -//M*/ + // + // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. + // + // By downloading, copying, installing or using the software you agree to this license. + // If you do not agree to this license, do not download, install, + // copy or use the software. + // + // + // License Agreement + // For Open Source Computer Vision Library + // (3-clause BSD License) + // + // Copyright (C) 2015, OpenCV Foundation, all rights reserved. + // Third party copyrights are property of their respective owners. + // + // Redistribution and use in source and binary forms, with or without modification, + // are permitted provided that the following conditions are met: + // + // * Redistribution's of source code must retain the above copyright notice, + // this list of conditions and the following disclaimer. + // + // * Redistribution's in binary form must reproduce the above copyright notice, + // this list of conditions and the following disclaimer in the documentation + // and/or other materials provided with the distribution. + // + // * The name of the copyright holders may not be used to endorse or promote products + // derived from this software without specific prior written permission. + // + // This software is provided by the copyright holders and contributors "as is" and + // any express or implied warranties, including, but not limited to, the implied + // warranties of merchantability and fitness for a particular purpose are disclaimed. + // In no event shall the Intel Corporation or contributors be liable for any direct, + // indirect, incidental, special, exemplary, or consequential damages + // (including, but not limited to, procurement of substitute goods or services; + // loss of use, data, or profits; or business interruption) however caused + // and on any theory of liability, whether in contract, strict liability, + // or tort (including negligence or otherwise) arising in any way out of + // the use of this software, even if advised of the possibility of such damage. + // + //M*/ + + #ifndef __OPENCV_OBSTRUCTION_FREE_CPP__ #define __OPENCV_OBSTRUCTION_FREE_CPP__ -#ifdef __cplusplus #include #include @@ -48,7 +52,6 @@ #include #include -#include namespace cv { @@ -63,18 +66,17 @@ void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ /////////construct image pyramids////// int pyramid_level=3; - std::vector video_coarseLeve; + std::vector video_coarseLevel; for (size_t frame_i=0; frame_i Date: Thu, 22 Jun 2017 01:30:19 +0900 Subject: [PATCH 05/17] fix typo --- modules/xphoto/src/obstruction_free.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index b5403cb47a5..b0d504ca522 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -69,7 +69,7 @@ void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ std::vector video_coarseLevel; for (size_t frame_i=0; frame_i Date: Wed, 19 Jul 2017 00:12:47 +0900 Subject: [PATCH 06/17] Initialization part --- .../opencv2/xphoto/obstruction_free.hpp | 112 +++--- modules/xphoto/src/obstruction_free.cpp | 369 +++++++++++++++--- 2 files changed, 376 insertions(+), 105 deletions(-) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 55c08f215bb..076e4ddd538 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -1,44 +1,6 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // (3-clause BSD License) - // - // Copyright (C) 2015, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. #ifndef __OPENCV_OBSTRUCTION_FREE_HPP__ @@ -50,6 +12,10 @@ */ #include +#include +#include +#include +#include namespace cv { @@ -60,19 +26,71 @@ namespace xphoto //! @{ - /** @brief The function implements a general obstruction free approach that can remove occlusions and reflections from input image sequences without manual masks. + /** @brief The class implements a general obstruction free approach that can remove occlusions and reflections from input image sequences without manual masks. See the original paper @cite Xue2015ObstructionFree for more details. - @param srcImgs source image sequences, involving translation motions. - @param dst Obstruction-removed destination image, corresponding to the reference image, with the same size and type. In general, the reference image is the center frame of the input image. - @param mask mask (CV_8UC1), where zero pixels indicate area to be estimated to be occlusions. + */ - CV_EXPORTS void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask); + class CV_EXPORTS obstructionFree + { + public: + /*! + * @brief Constructors + * @param srcImgs input image sequences + */ + obstructionFree(); + obstructionFree(const std::vector &srcImgs); + + /*! + * @brief core function to remove occlusions + * @param srcImgs source image sequences, involving translation motions. + * @param dst Obstruction-removed destination image, corresponding to the reference image, with the same size and type. In general, the reference image is the center frame of the input image. + * @param mask estimated occlusion areas (CV_8UC1), where zero pixels indicate area to be estimated to be occlusions. + * @param obstructionType: reflection (0) or opaque obstruction (1) + */ + void removeOcc(const std::vector &srcImgs, Mat &dst, Mat &mask, const int obstructionType); + + private: + /*! + * @brief Parameters + */ + size_t frameNumber; //frame number of the input sequences + size_t referenceNumber; //target frame + int pyramidLevel; // Pyramid level + int coarseIterations; // iteration number for the coarsest level + int upperIterations; // iteration number for the upper level + int fixedPointIterations; // during each level of the pyramid + int sorIterations; // iterations of SOR + float omega; // relaxation factor in SOR + float lambda1; // weight for alpha map smoothness constraints + float lambda2; // weight for image smoothness constraints + float lambda3; // weight for independence between back/foreground component + float lambda4; // weight for gradient sparsity + + + std::vector backFlowFields; //estimated optical flow fields in the background layer + std::vector foreFlowFields; //estimated optical flow fields in the foreground layer + std::vector warpedToReference; //warped images from input images through the estimated background flow fields + + //switch different methods. TODO + int interpolationType; //interpolation type: 0(reflection) or 1(opaque occlusion) + int edgeflowType; //edge flow type + + //private functions + //build pyramid by stacking all input image sequences + std::vector buildPyramid( const std::vector & srcImgs); + //extract certain level of image sequences from the stacked image pyramid + std::vector extractLevelImgs(const std::vector& pyramid, const int level); + //initialization: decompose the motion fields + void motionInitDirect(const std::vector& video_input, std::vector& back_Flowfields, std::vector& fore_flowfields, std::vector& warpedToReference); + //initialization: decompose the image components + Mat imgInitDecompose(const std::vector &warpedImgs, const std::vector& back_flowfields, const int obstructionType); + }; //! @} } } -#endif // __OPENCV_INPAINTING_HPP__ +#endif // __OPENCV_OBSTRUCTION_FREE_HPP__ diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index b0d504ca522..ffc50b55724 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -1,44 +1,6 @@ -/*M/////////////////////////////////////////////////////////////////////////////////////// - // - // IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. - // - // By downloading, copying, installing or using the software you agree to this license. - // If you do not agree to this license, do not download, install, - // copy or use the software. - // - // - // License Agreement - // For Open Source Computer Vision Library - // (3-clause BSD License) - // - // Copyright (C) 2015, OpenCV Foundation, all rights reserved. - // Third party copyrights are property of their respective owners. - // - // Redistribution and use in source and binary forms, with or without modification, - // are permitted provided that the following conditions are met: - // - // * Redistribution's of source code must retain the above copyright notice, - // this list of conditions and the following disclaimer. - // - // * Redistribution's in binary form must reproduce the above copyright notice, - // this list of conditions and the following disclaimer in the documentation - // and/or other materials provided with the distribution. - // - // * The name of the copyright holders may not be used to endorse or promote products - // derived from this software without specific prior written permission. - // - // This software is provided by the copyright holders and contributors "as is" and - // any express or implied warranties, including, but not limited to, the implied - // warranties of merchantability and fitness for a particular purpose are disclaimed. - // In no event shall the Intel Corporation or contributors be liable for any direct, - // indirect, incidental, special, exemplary, or consequential damages - // (including, but not limited to, procurement of substitute goods or services; - // loss of use, data, or profits; or business interruption) however caused - // and on any theory of liability, whether in contract, strict liability, - // or tort (including negligence or otherwise) arising in any way out of - // the use of this software, even if advised of the possibility of such damage. - // - //M*/ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. @@ -49,34 +11,325 @@ #include #include - -#include -#include +#include "obstructionFree.hpp" namespace cv { namespace xphoto { -void obstructionFree(const std::vector &srcImgs, Mat &dst, Mat &mask){ - size_t frameNumber = srcImgs.size(); - size_t reference_number = (frameNumber-1)/2;; - dst.create( srcImgs[reference_number].size(), srcImgs[reference_number].type() ); - mask = Mat::zeros(dst.rows,dst.cols,CV_8UC1); - /////////construct image pyramids////// - int pyramid_level=3; - std::vector video_coarseLevel; +//constructor +obstructionFree::obstructionFree() +{ + //parameter + pyramidLevel = 3; // Pyramid level + coarseIterations = 4; // iteration number for the coarsest level + upperIterations = 1; // iteration number for the upper level + fixedPointIterations = 5; // during each level of the pyramid + sorIterations = 25; // iterations of SOR + omega = 1.6f; // relaxation factor in SOR + lambda1 = 1.0f; // weight for alpha map smoothness constraints + lambda2 = 0.1f; // weight for image smoothness constraints + lambda3 = 3000.0f; // weight for independence between back/foreground component + lambda4 = 0.5f; // weight for gradient sparsity +} + +obstructionFree::obstructionFree(const std::vector &srcImgs) +{ + //parameter + frameNumber = srcImgs.size(); + referenceNumber=(frameNumber-1)/2; //target frame + pyramidLevel = 3; // Pyramid level + coarseIterations = 4; // iteration number for the coarsest level + upperIterations = 1; // iteration number for the upper level + fixedPointIterations = 5; // during each level of the pyramid + sorIterations = 25; // iterations of SOR + omega = 1.6f; // relaxation factor in SOR + lambda1 = 1.0f; // weight for alpha map smoothness constraints + lambda2 = 0.1f; // weight for image smoothness constraints + lambda3 = 3000.0f; // weight for independence between back/foreground component + lambda4 = 0.5f; // weight for gradient sparsity +} + +//build pyramid by stacking all input image sequences +std::vector obstructionFree::buildPyramid(const std::vector & srcImgs) +{ + std::vector pyramid; + + for (size_t frame_i=0; frame_iframeNumber; frame_i++){ + Mat grey; + cvtColor(srcImgs[frame_i], grey, COLOR_RGB2GRAY); + pyramid.push_back(grey.clone()); + } + + Mat thisLevel, nextLevel; + int thisLevelId; + for (int level_i=1; level_ipyramidLevel; level_i++){ + for (size_t frame_i=0; frame_iframeNumber; frame_i++){ + thisLevelId=(level_i-1)*this->frameNumber+frame_i; + //nextLevelId=i*this.pyramidLevel+frame_i; + thisLevel=pyramid[thisLevelId]; + pyrDown(thisLevel, nextLevel); + pyramid.push_back(nextLevel.clone()); + } + } + return pyramid; +} + +//extract certain level of image sequences from the stacked image pyramid +std::vector obstructionFree::extractLevelImgs(const std::vector& pyramid, const int level){ + std::vector levelPyramid; + int imgId; + for (size_t frame_i=0; frame_iframeNumber; frame_i++){ + imgId=level*this->frameNumber+frame_i; + levelPyramid.push_back(pyramid[imgId].clone()); + } + return levelPyramid; +} + + Mat indexToMask(Mat indexMat, int rows, int cols){ + Mat maskMat=Mat::zeros(rows, cols, CV_8UC1); + for (int i = 0; i < indexMat.cols; i++ ) { + for (int j = 0; j < indexMat.rows; j++) { + Vec2i mask_loca = indexMat.at(j, i); + if (mask_loca[0] !=0 && mask_loca[1] !=0) { + maskMat.at(Point(mask_loca)) = 255;} + }} + return maskMat; + } + +//estimate homography matrix using edge flow fields +Mat flowHomography(Mat edges, Mat flow, int ransacThre){ + Mat inlierMask, inlier_edges, inilier_edgeLocations; + std::vector edge_Locations; + + findNonZero(edges, edge_Locations); + + std::vector obj_edgeflow; + + for(size_t i = 0; i(src_y, src_x); + obj_edgeflow.push_back(Point2i(src_x + f.x, src_y + f.y)); + } + + Mat Homography = findHomography( edge_Locations, obj_edgeflow, RANSAC, ransacThre, inlierMask); + + Mat(edge_Locations).copyTo(inilier_edgeLocations,inlierMask); + + //convert index matrix to mask matrix + inlier_edges=indexToMask(inilier_edgeLocations, edges.rows, edges.cols); + + return inlier_edges; +} + +Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ + Mat denseFlow; + std::vector sparseFrom; + std::vector sparseTo; + + std::vector edge_Location; + findNonZero(im1_edges, edge_Location); + for(size_t i = 0; i(src_y, src_x); + sparseTo.push_back(Point2f(src_x + f.x, src_y + f.y)); + } + + Ptr epicInterpolation=ximgproc::createEdgeAwareInterpolator(); + epicInterpolation->interpolate(im1,sparseFrom,im2,sparseTo,denseFlow); + return denseFlow; +} + +//show motion fields using color +void colorFlow(Mat flow, std::string figName="optical flow") +{ + //extraxt x and y channels + Mat xy[2]; //X,Y + split(flow, xy); + + //calculate angle and magnitude + Mat magnitude, angle; + cartToPolar(xy[0], xy[1], magnitude, angle, true); + + //translate magnitude to range [0;1] + double mag_max; + minMaxLoc(magnitude, 0, &mag_max); + magnitude.convertTo(magnitude, -1, 1.0 / mag_max); + + //build hsv image + Mat _hsv[3], hsv; + _hsv[0] = angle; + _hsv[1] = Mat::ones(angle.size(), CV_32F); + _hsv[2] = magnitude; + merge(_hsv, 3, hsv); + + //convert to BGR and show + Mat bgr;//CV_32FC3 matrix + cvtColor(hsv, bgr, COLOR_HSV2BGR); + imshow(figName, bgr); +} + +//motion decomposition between two images: target frame and source frame +void initMotionDecompose(Mat im1, Mat im2, Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre=1, int fore_ransacThre=1){ + if (im1.channels()!= 1) + cvtColor(im1, im1, COLOR_RGB2GRAY); + if (im2.channels()!= 1) + cvtColor(im2, im2, COLOR_RGB2GRAY); + + Mat im1_edge, im2_edge; + Mat flow; + Mat edgeflow; //extracted edgeflow + + //Mat backH, mask_backH; + Mat back_edges, rest_edges, fore_edges; //edges aligned to the back layer using homography, remaining layer, foreground layers + Mat back_flow, rest_flow, fore_flow; + + + Canny(im1, im1_edge, 10, 100,3,true); + Canny(im2, im2_edge, 10, 100,3,true); + + ///////////////replace edgeflow + Ptr deepflow = optflow::createOptFlow_DeepFlow(); + deepflow->calc(im1, im2, flow); + //colorFlow(flow,"optical_flow"); + flow.copyTo(edgeflow, im1_edge); + //colorFlow(edgeflow,"edge_flow"); + +////////flow=>points using homography-ransac filtering, and then extract flow on the filtered edges + back_edges=flowHomography(im1_edge, edgeflow, back_ransacThre); + //imshow("back_edges", back_edges); + edgeflow.copyTo(back_flow,back_edges); + //colorFlow(back_flow, "back_flow"); + //////////rest edges and flows + rest_edges=im1_edge-back_edges; + //imshow("rest_edges", rest_edges); + rest_flow=edgeflow-back_flow; + //colorFlow(rest_flow, "rest_flow"); + + ////////////align resting flows to another homograghy + fore_edges=flowHomography(rest_edges, rest_flow, fore_ransacThre); + //imshow("fore_edges", fore_edges); + rest_flow.copyTo(fore_flow,fore_edges); + //colorFlow(fore_flow, "fore_flow"); + +///////////////////interpolation from sparse edgeFlow to denseFlow///////////////////// + back_denseFlow=sparseToDense(im1, im2, back_edges, back_flow); + fore_denseFlow=sparseToDense(im1, im2, fore_edges, fore_flow); + //colorFlow(back_denseFlow,"inter_back_denseflow"); + //colorFlow(fore_denseFlow,"inter_fore_denseflow"); +} + +//warping im1 to output through optical flow: +//flow=flow->cal(im1,im2), so warp im2 to back +Mat imgWarpFlow(Mat im1, Mat flow){ + Mat flowmap_x(flow.size(), CV_32FC1); + Mat flowmap_y(flow.size(), CV_32FC1); + for (int j = 0; j < flowmap_x.rows; j++){ + for (int i = 0; i < flowmap_x.cols; ++i){ + Point2f f = flow.at(j, i); + flowmap_y.at(j, i) = float(j + f.y); + flowmap_x.at(j, i) = float(i + f.x); + }} + Mat warpedFrame; + remap(im1, warpedFrame, flowmap_x,flowmap_y ,INTER_CUBIC,BORDER_CONSTANT,255); + return warpedFrame; +} + +//Initialization: decompose the motion fields +void obstructionFree::motionInitDirect(const std::vector& video_input, std::vector& back_flowfields, std::vector& fore_flowfields, std::vector& warpedToReference){ + int back_ransacThre=1; + int fore_ransacThre=1; + for (size_t frame_i=0; frame_i &warpedImgs, const std::vector& back_flowfields, const int obstructionType){ + Mat background; + //if occlusion is reflection type + if (obstructionType==0){ + background=warpedImgs[referenceNumber]; + for (size_t frame_i=0; frame_i &srcImgs, Mat &dst, Mat &mask, const int obstructionType){ + //initialization + std::vector srcPyramid=buildPyramid(srcImgs); + std::vector coarseLevel = extractLevelImgs(srcPyramid,pyramidLevel-1); + motionInitDirect(coarseLevel, backFlowFields, foreFlowFields, warpedToReference); + Mat initBack; + initBack=imgInitDecompose(warpedToReference,backFlowFields,1); + + //alternative optimization:TODO + + + +} + + + } } #endif // __OPENCV_OBSTRUCTION_FREE_CPP__ From 1974f45cf54803456c35f2908f3d4debdeb7f1fc Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 11:06:16 +0900 Subject: [PATCH 07/17] add dependency of outflow, opencv_ximgproc, and calib3d, where outflow and opencv_ximgproc may be removed later. --- modules/xphoto/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/xphoto/CMakeLists.txt b/modules/xphoto/CMakeLists.txt index 3e550a3c892..ebe81840b4b 100644 --- a/modules/xphoto/CMakeLists.txt +++ b/modules/xphoto/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Addon to basic photo module") -ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python) +ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python optflow opencv_ximgproc calib3d) From a18e2ece38aa153f59fb8bab804e0bd964e0e64c Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 11:11:27 +0900 Subject: [PATCH 08/17] add dependency of opencv_optflow, opencv_ximgproc, and opencv_calib3d, where opencv_optflow and opencv_ximgproc may be removed later. --- modules/xphoto/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/xphoto/CMakeLists.txt b/modules/xphoto/CMakeLists.txt index ebe81840b4b..123944113a3 100644 --- a/modules/xphoto/CMakeLists.txt +++ b/modules/xphoto/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Addon to basic photo module") -ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python optflow opencv_ximgproc calib3d) +ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python opencv_optflow opencv_ximgproc opencv_calib3d) From 63c641db05215323e3bee4c0f9d9899466074357 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 11:11:27 +0900 Subject: [PATCH 09/17] add dependency of opencv_optflow, opencv_ximgproc, and opencv_calib3d, where opencv_optflow and opencv_ximgproc may be removed later. --- modules/xphoto/CMakeLists.txt | 2 +- .../xphoto/include/opencv2/xphoto/obstruction_free.hpp | 8 +++++++- modules/xphoto/src/obstruction_free.cpp | 2 -- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/modules/xphoto/CMakeLists.txt b/modules/xphoto/CMakeLists.txt index ebe81840b4b..3a2a170988c 100644 --- a/modules/xphoto/CMakeLists.txt +++ b/modules/xphoto/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Addon to basic photo module") -ocv_define_module(xphoto opencv_core opencv_imgproc WRAP python optflow opencv_ximgproc calib3d) +ocv_define_module(xphoto opencv_core opencv_imgproc opencv_optflow opencv_ximgproc opencv_calib3d opencv_highgui WRAP python) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 076e4ddd538..34a395db065 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -17,6 +17,8 @@ #include #include +#include + namespace cv { namespace xphoto @@ -37,9 +39,13 @@ namespace xphoto public: /*! * @brief Constructors - * @param srcImgs input image sequences */ obstructionFree(); + + /*! + * @brief Constructors + * @param srcImgs input image sequences + */ obstructionFree(const std::vector &srcImgs); /*! diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index ffc50b55724..8836f56721e 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -257,8 +257,6 @@ void obstructionFree::motionInitDirect(const std::vector& video_input, std: //decompose motion fields into fore/background initMotionDecompose(im1, im2, back_denseFlow, fore_denseFlow, back_ransacThre, fore_ransacThre); - - //cout< Date: Wed, 19 Jul 2017 14:15:11 +0900 Subject: [PATCH 10/17] amend typo and dependency --- modules/xphoto/CMakeLists_LOCAL_27080.txt | 2 -- .../include/opencv2/xphoto/obstruction_free.hpp | 12 ++++++------ modules/xphoto/src/obstruction_free.cpp | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) delete mode 100644 modules/xphoto/CMakeLists_LOCAL_27080.txt diff --git a/modules/xphoto/CMakeLists_LOCAL_27080.txt b/modules/xphoto/CMakeLists_LOCAL_27080.txt deleted file mode 100644 index 3a2a170988c..00000000000 --- a/modules/xphoto/CMakeLists_LOCAL_27080.txt +++ /dev/null @@ -1,2 +0,0 @@ -set(the_description "Addon to basic photo module") -ocv_define_module(xphoto opencv_core opencv_imgproc opencv_optflow opencv_ximgproc opencv_calib3d opencv_highgui WRAP python) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 34a395db065..f3fb838cb8d 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -11,13 +11,13 @@ @author Binbin Xu */ -#include -#include -#include -#include -#include +#include "opencv2/core.hpp" +#include "opencv2/imgproc.hpp" +#include "opencv2/ximgproc/sparse_match_interpolator.hpp" +#include "opencv2/optflow.hpp" +#include "opencv2/calib3d.hpp" -#include +#include "opencv2/highgui.hpp" namespace cv { diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 8836f56721e..03ce757b9f5 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -11,7 +11,7 @@ #include #include -#include "obstructionFree.hpp" +#include "obstruction_free.hpp" namespace cv { From c694e46e5afb8a759cf525ed1afd29c1725b71ec Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 14:20:01 +0900 Subject: [PATCH 11/17] amend head file dependency --- modules/xphoto/src/obstruction_free.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 03ce757b9f5..023cf207965 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -11,7 +11,6 @@ #include #include -#include "obstruction_free.hpp" namespace cv { From 0bca4caebc7f91ba29e2034dc056a32f979e2081 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 16:46:47 +0900 Subject: [PATCH 12/17] fix warnings and errors --- .../opencv2/xphoto/obstruction_free.hpp | 56 +++++++++--- modules/xphoto/src/obstruction_free.cpp | 90 ++++++++++--------- 2 files changed, 94 insertions(+), 52 deletions(-) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index f3fb838cb8d..6181562068f 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -52,10 +52,11 @@ namespace xphoto * @brief core function to remove occlusions * @param srcImgs source image sequences, involving translation motions. * @param dst Obstruction-removed destination image, corresponding to the reference image, with the same size and type. In general, the reference image is the center frame of the input image. + * @param foreground estimated reflection or opaque obstruction layer * @param mask estimated occlusion areas (CV_8UC1), where zero pixels indicate area to be estimated to be occlusions. * @param obstructionType: reflection (0) or opaque obstruction (1) */ - void removeOcc(const std::vector &srcImgs, Mat &dst, Mat &mask, const int obstructionType); + void removeOcc(const std::vector &srcImgs, Mat &dst, Mat& foreground, Mat &mask, const int obstructionType); private: /*! @@ -77,25 +78,60 @@ namespace xphoto std::vector backFlowFields; //estimated optical flow fields in the background layer std::vector foreFlowFields; //estimated optical flow fields in the foreground layer - std::vector warpedToReference; //warped images from input images through the estimated background flow fields + //std::vector warpedToReference; //warped images from input images through the estimated background flow fields //switch different methods. TODO - int interpolationType; //interpolation type: 0(reflection) or 1(opaque occlusion) - int edgeflowType; //edge flow type + //int interpolationType; //interpolation type: 0(reflection) or 1(opaque occlusion) + //int edgeflowType; //edge flow type - //private functions - //build pyramid by stacking all input image sequences + /** @brief private functions + */ + /** @brief Build pyramid by stacking all input image sequences + */ std::vector buildPyramid( const std::vector & srcImgs); - //extract certain level of image sequences from the stacked image pyramid + + /** @brief Extract certain level of image sequences from the stacked image pyramid + */ std::vector extractLevelImgs(const std::vector& pyramid, const int level); - //initialization: decompose the motion fields + + /** @brief Initialization: decompose the motion fields + */ void motionInitDirect(const std::vector& video_input, std::vector& back_Flowfields, std::vector& fore_flowfields, std::vector& warpedToReference); - //initialization: decompose the image components - Mat imgInitDecompose(const std::vector &warpedImgs, const std::vector& back_flowfields, const int obstructionType); + /** @brief Initialization: decompose the image components in the case of reflection + */ + Mat imgInitDecomRef(const std::vector &warpedImgs); + + /** @brief Initialization: decompose the image components in the case of opaque reflection + */ + Mat imgInitDecomOpaq(const std::vector &warpedImgs, Mat& foreground, Mat& alphaMap); }; //! @} +/** @brief Convert from sparse edge displacement to dense motion fields + */ +CV_EXPORTS Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow); + +/** @brief Visualize the optical flow flow with the window named figName + */ +CV_EXPORTS void colorFlow(const Mat& flow, std::string figName); + +/** @brief Decompose motion between two images: target frame and source frame, using homography ransac + */ +CV_EXPORTS void initMotionDecompose(const Mat& im1 , const Mat& im2 , Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre, int fore_ransacThre); + +/** @brief Warp im1 to output through optical flow: +*/ +CV_EXPORTS Mat imgWarpFlow(const Mat& im1, const Mat& flow); + +/** @brief Estimate homography matrix using edge flow fields +*/ +CV_EXPORTS Mat flowHomography(const Mat& edges, const Mat& flow, const int ransacThre); + +/** @brief Convert from index to matrix +*/ +CV_EXPORTS Mat indexToMask(const Mat& indexMat, const int rows, const int cols); + } } diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 023cf207965..02d8fae71e7 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -63,7 +63,7 @@ std::vector obstructionFree::buildPyramid(const std::vector & srcImgs) } Mat thisLevel, nextLevel; - int thisLevelId; + size_t thisLevelId; for (int level_i=1; level_ipyramidLevel; level_i++){ for (size_t frame_i=0; frame_iframeNumber; frame_i++){ thisLevelId=(level_i-1)*this->frameNumber+frame_i; @@ -87,16 +87,16 @@ std::vector obstructionFree::extractLevelImgs(const std::vector& pyram return levelPyramid; } - Mat indexToMask(Mat indexMat, int rows, int cols){ - Mat maskMat=Mat::zeros(rows, cols, CV_8UC1); - for (int i = 0; i < indexMat.cols; i++ ) { - for (int j = 0; j < indexMat.rows; j++) { - Vec2i mask_loca = indexMat.at(j, i); - if (mask_loca[0] !=0 && mask_loca[1] !=0) { - maskMat.at(Point(mask_loca)) = 255;} - }} +Mat indexToMask(Mat indexMat, int rows, int cols){ + Mat maskMat=Mat::zeros(rows, cols, CV_8UC1); + for (int i = 0; i < indexMat.cols; i++ ) { + for (int j = 0; j < indexMat.rows; j++) { + Vec2i mask_loca = indexMat.at(j, i); + if (mask_loca[0] !=0 && mask_loca[1] !=0) { + maskMat.at(Point(mask_loca)) = 255;} + }} return maskMat; - } +} //estimate homography matrix using edge flow fields Mat flowHomography(Mat edges, Mat flow, int ransacThre){ @@ -108,8 +108,8 @@ Mat flowHomography(Mat edges, Mat flow, int ransacThre){ std::vector obj_edgeflow; for(size_t i = 0; i(src_y, src_x); obj_edgeflow.push_back(Point2i(src_x + f.x, src_y + f.y)); } @@ -132,8 +132,8 @@ Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ std::vector edge_Location; findNonZero(im1_edges, edge_Location); for(size_t i = 0; i(src_y, src_x); sparseTo.push_back(Point2f(src_x + f.x, src_y + f.y)); @@ -275,36 +275,37 @@ void obstructionFree::motionInitDirect(const std::vector& video_input, std: } } -//initialization: decompose the image components -Mat obstructionFree::imgInitDecompose(const std::vector &warpedImgs, const std::vector& back_flowfields, const int obstructionType){ +//initialization: decompose the image components in the case of reflection +Mat obstructionFree::imgInitDecomRef(const std::vector &warpedImgs){ Mat background; - //if occlusion is reflection type - if (obstructionType==0){ - background=warpedImgs[referenceNumber]; - for (size_t frame_i=0; frame_i &warpedImgs, Mat& foreground, Mat& alphaMap){ + Mat background; + Mat sum=Mat::zeros(warpedImgs[referenceNumber].rows,warpedImgs[referenceNumber].cols,CV_32F); + Mat temp,background_temp; + for (size_t frame_i=0; frame_i &warpedImgs, const //core function void obstructionFree::removeOcc(const std::vector &srcImgs, Mat &dst, Mat &mask, const int obstructionType){ //initialization + std::vector warpedToReference; std::vector srcPyramid=buildPyramid(srcImgs); std::vector coarseLevel = extractLevelImgs(srcPyramid,pyramidLevel-1); motionInitDirect(coarseLevel, backFlowFields, foreFlowFields, warpedToReference); - Mat initBack; - initBack=imgInitDecompose(warpedToReference,backFlowFields,1); + + CV_Assert(obstructionType==0 || obstructionType==1); + if (obstructionType==0) + dst=imgInitDecomRef(warpedToReference); + else + dst=imgInitDecomOpaq(warpedToReference,foreground,mask); //alternative optimization:TODO From 0b426a284b6083a1f2ab5544fede46b55d62bc99 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 18:33:06 +0900 Subject: [PATCH 13/17] fix declarations and type conversions --- .../opencv2/xphoto/obstruction_free.hpp | 41 ++++++++++--------- modules/xphoto/src/obstruction_free.cpp | 22 +++++----- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 6181562068f..63e8009ec9d 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -105,32 +105,33 @@ namespace xphoto /** @brief Initialization: decompose the image components in the case of opaque reflection */ Mat imgInitDecomOpaq(const std::vector &warpedImgs, Mat& foreground, Mat& alphaMap); - }; -//! @} -/** @brief Convert from sparse edge displacement to dense motion fields - */ -CV_EXPORTS Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow); + /** @brief Convert from sparse edge displacement to dense motion fields + */ + Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow); -/** @brief Visualize the optical flow flow with the window named figName - */ -CV_EXPORTS void colorFlow(const Mat& flow, std::string figName); + /** @brief Visualize the optical flow flow with the window named figName + */ + void colorFlow(const Mat& flow, std::string figName); -/** @brief Decompose motion between two images: target frame and source frame, using homography ransac - */ -CV_EXPORTS void initMotionDecompose(const Mat& im1 , const Mat& im2 , Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre, int fore_ransacThre); + /** @brief Decompose motion between two images: target frame and source frame, using homography ransac + */ + void initMotionDecompose(const Mat& im1 , const Mat& im2 , Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre, int fore_ransacThre); -/** @brief Warp im1 to output through optical flow: -*/ -CV_EXPORTS Mat imgWarpFlow(const Mat& im1, const Mat& flow); + /** @brief Warp im1 to output through optical flow: + */ + Mat imgWarpFlow(const Mat& im1, const Mat& flow); -/** @brief Estimate homography matrix using edge flow fields -*/ -CV_EXPORTS Mat flowHomography(const Mat& edges, const Mat& flow, const int ransacThre); + /** @brief Estimate homography matrix using edge flow fields + */ + Mat flowHomography(const Mat& edges, const Mat& flow, const int ransacThre); + + /** @brief Convert from index to matrix + */ + Mat indexToMask(const Mat& indexMat, const int rows, const int cols); + }; +//! @} -/** @brief Convert from index to matrix -*/ -CV_EXPORTS Mat indexToMask(const Mat& indexMat, const int rows, const int cols); } } diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 02d8fae71e7..f422829c3f8 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -79,7 +79,7 @@ std::vector obstructionFree::buildPyramid(const std::vector & srcImgs) //extract certain level of image sequences from the stacked image pyramid std::vector obstructionFree::extractLevelImgs(const std::vector& pyramid, const int level){ std::vector levelPyramid; - int imgId; + size_t imgId; for (size_t frame_i=0; frame_iframeNumber; frame_i++){ imgId=level*this->frameNumber+frame_i; levelPyramid.push_back(pyramid[imgId].clone()); @@ -87,7 +87,7 @@ std::vector obstructionFree::extractLevelImgs(const std::vector& pyram return levelPyramid; } -Mat indexToMask(Mat indexMat, int rows, int cols){ +Mat obstructionFree::indexToMask(Mat indexMat, int rows, int cols){ Mat maskMat=Mat::zeros(rows, cols, CV_8UC1); for (int i = 0; i < indexMat.cols; i++ ) { for (int j = 0; j < indexMat.rows; j++) { @@ -99,7 +99,7 @@ Mat indexToMask(Mat indexMat, int rows, int cols){ } //estimate homography matrix using edge flow fields -Mat flowHomography(Mat edges, Mat flow, int ransacThre){ +Mat obstructionFree::flowHomography(Mat edges, Mat flow, int ransacThre){ Mat inlierMask, inlier_edges, inilier_edgeLocations; std::vector edge_Locations; @@ -108,8 +108,8 @@ Mat flowHomography(Mat edges, Mat flow, int ransacThre){ std::vector obj_edgeflow; for(size_t i = 0; i(src_y, src_x); obj_edgeflow.push_back(Point2i(src_x + f.x, src_y + f.y)); } @@ -124,7 +124,7 @@ Mat flowHomography(Mat edges, Mat flow, int ransacThre){ return inlier_edges; } -Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ +Mat obstructionFree::sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ Mat denseFlow; std::vector sparseFrom; std::vector sparseTo; @@ -134,9 +134,9 @@ Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ for(size_t i = 0; i(src_y, src_x); - sparseTo.push_back(Point2f(src_x + f.x, src_y + f.y)); + sparseTo.push_back(Point2f(float(src_x + f.x), float(src_y + f.y))); } Ptr epicInterpolation=ximgproc::createEdgeAwareInterpolator(); @@ -145,7 +145,7 @@ Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ } //show motion fields using color -void colorFlow(Mat flow, std::string figName="optical flow") +void obstructionFree::colorFlow(Mat flow, std::string figName="optical flow") { //extraxt x and y channels Mat xy[2]; //X,Y @@ -174,7 +174,7 @@ void colorFlow(Mat flow, std::string figName="optical flow") } //motion decomposition between two images: target frame and source frame -void initMotionDecompose(Mat im1, Mat im2, Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre=1, int fore_ransacThre=1){ +void obstructionFree::initMotionDecompose(Mat im1, Mat im2, Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre=1, int fore_ransacThre=1){ if (im1.channels()!= 1) cvtColor(im1, im1, COLOR_RGB2GRAY); if (im2.channels()!= 1) @@ -225,7 +225,7 @@ void initMotionDecompose(Mat im1, Mat im2, Mat& back_denseFlow, Mat& fore_denseF //warping im1 to output through optical flow: //flow=flow->cal(im1,im2), so warp im2 to back -Mat imgWarpFlow(Mat im1, Mat flow){ +Mat obstructionFree::imgWarpFlow(Mat im1, Mat flow){ Mat flowmap_x(flow.size(), CV_32FC1); Mat flowmap_y(flow.size(), CV_32FC1); for (int j = 0; j < flowmap_x.rows; j++){ From fdac8cdf72ca163545aee980153d6f0c607fa221 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Wed, 19 Jul 2017 20:21:17 +0900 Subject: [PATCH 14/17] fix the mistakes in copying from local files --- .../opencv2/xphoto/obstruction_free.hpp | 2 +- modules/xphoto/src/obstruction_free.cpp | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 63e8009ec9d..41235aa6142 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -108,7 +108,7 @@ namespace xphoto /** @brief Convert from sparse edge displacement to dense motion fields */ - Mat sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow); + Mat sparseToDense(const Mat& im1, const Mat& im2, const Mat& im1_edges, const Mat& sparseFlow); /** @brief Visualize the optical flow flow with the window named figName */ diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index f422829c3f8..8dc4b3a986d 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -87,7 +87,7 @@ std::vector obstructionFree::extractLevelImgs(const std::vector& pyram return levelPyramid; } -Mat obstructionFree::indexToMask(Mat indexMat, int rows, int cols){ +Mat obstructionFree::indexToMask(const Mat& indexMat, const int rows, const int cols){ Mat maskMat=Mat::zeros(rows, cols, CV_8UC1); for (int i = 0; i < indexMat.cols; i++ ) { for (int j = 0; j < indexMat.rows; j++) { @@ -99,7 +99,7 @@ Mat obstructionFree::indexToMask(Mat indexMat, int rows, int cols){ } //estimate homography matrix using edge flow fields -Mat obstructionFree::flowHomography(Mat edges, Mat flow, int ransacThre){ +Mat obstructionFree::flowHomography(const Mat& edges, const Mat& flow, const int ransacThre){ Mat inlierMask, inlier_edges, inilier_edgeLocations; std::vector edge_Locations; @@ -111,7 +111,7 @@ Mat obstructionFree::flowHomography(Mat edges, Mat flow, int ransacThre){ int src_x=edge_Locations[i].x; int src_y=edge_Locations[i].y; Point2f f = flow.at(src_y, src_x); - obj_edgeflow.push_back(Point2i(src_x + f.x, src_y + f.y)); + obj_edgeflow.push_back(Point2f(src_x + f.x, src_y + f.y)); } Mat Homography = findHomography( edge_Locations, obj_edgeflow, RANSAC, ransacThre, inlierMask); @@ -124,7 +124,7 @@ Mat obstructionFree::flowHomography(Mat edges, Mat flow, int ransacThre){ return inlier_edges; } -Mat obstructionFree::sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFlow){ +Mat obstructionFree::sparseToDense(const Mat& im1, const Mat& im2, const Mat& im1_edges, const Mat& sparseFlow){ Mat denseFlow; std::vector sparseFrom; std::vector sparseTo; @@ -145,7 +145,7 @@ Mat obstructionFree::sparseToDense(Mat im1, Mat im2, Mat im1_edges, Mat sparseFl } //show motion fields using color -void obstructionFree::colorFlow(Mat flow, std::string figName="optical flow") +void obstructionFree::colorFlow(const Mat& flow, std::string figName="optical flow") { //extraxt x and y channels Mat xy[2]; //X,Y @@ -174,7 +174,7 @@ void obstructionFree::colorFlow(Mat flow, std::string figName="optical flow") } //motion decomposition between two images: target frame and source frame -void obstructionFree::initMotionDecompose(Mat im1, Mat im2, Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre=1, int fore_ransacThre=1){ +void obstructionFree::initMotionDecompose(const Mat& im1, const Mat& im2, Mat& back_denseFlow, Mat& fore_denseFlow, int back_ransacThre=1, int fore_ransacThre=1){ if (im1.channels()!= 1) cvtColor(im1, im1, COLOR_RGB2GRAY); if (im2.channels()!= 1) @@ -225,7 +225,7 @@ void obstructionFree::initMotionDecompose(Mat im1, Mat im2, Mat& back_denseFlow, //warping im1 to output through optical flow: //flow=flow->cal(im1,im2), so warp im2 to back -Mat obstructionFree::imgWarpFlow(Mat im1, Mat flow){ +Mat obstructionFree::imgWarpFlow(const Mat& im1, const Mat& flow){ Mat flowmap_x(flow.size(), CV_32FC1); Mat flowmap_y(flow.size(), CV_32FC1); for (int j = 0; j < flowmap_x.rows; j++){ @@ -312,7 +312,7 @@ Mat obstructionFree::imgInitDecomOpaq(const std::vector &warpedImgs, Mat& //core function -void obstructionFree::removeOcc(const std::vector &srcImgs, Mat &dst, Mat &mask, const int obstructionType){ +void obstructionFree::removeOcc(const std::vector &srcImgs, Mat &dst, Mat& foreground, Mat &mask, const int obstructionType){ //initialization std::vector warpedToReference; std::vector srcPyramid=buildPyramid(srcImgs); @@ -328,11 +328,11 @@ void obstructionFree::removeOcc(const std::vector &srcImgs, Mat &dst, Mat //alternative optimization:TODO - } + } } #endif // __OPENCV_OBSTRUCTION_FREE_CPP__ From 5835871a94d426a8731f67b4b3ad9ee6a5b07cb0 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Sat, 22 Jul 2017 01:50:32 +0900 Subject: [PATCH 15/17] add weights calculation parts --- .../opencv2/xphoto/obstruction_free.hpp | 14 ++++ modules/xphoto/src/obstruction_free.cpp | 68 +++++++++++++++++++ 2 files changed, 82 insertions(+) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index 41235aa6142..d0f060b1d40 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -129,6 +129,20 @@ namespace xphoto /** @brief Convert from index to matrix */ Mat indexToMask(const Mat& indexMat, const int rows, const int cols); + + /*! + * @brief Calculate the weights in the alternative motion decomposition step + * @param inputSequence Input sequences + * @param background Estimated background component in the last iteration + * @param foreground Estimated foreground/obstruction component in the last iteration + * @param alphaMap Estimated alpha map for obstruction layer + * @return omega_1 The weight omega_1 = \phi(||I^t - W_O^t\hat{I}_O - W_O^t\hat{A} \circ W_B^t\hat{I}_B||^2)^{-1} + * @return omega_2 The weight omega_2 = \phi(||D_x\hat{I}_B||^2+||D_y\hat{I}_B||^2)^{-1} + * @return omega_3 The weight omega_3 = \phi(||D_x\hat{I}_O||^2+||D_y\hat{I}_O||^2)^{-1} + */ + void motDecomIrlsWeight(const std::vector& inputSequence, const Mat& background, const Mat& foreground, + Mat& alphaMap, std::vector& omega_1, std::vector& omega_2, std::vector& omega_3); + }; //! @} diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index 8dc4b3a986d..dec0e563126 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -310,6 +310,74 @@ Mat obstructionFree::imgInitDecomOpaq(const std::vector &warpedImgs, Mat& return background; } +//Calculate the weights in the alternative motion decomposition step +void obstructionFree::motDecomIrlsWeight(const std::vector& inputSequence, const Mat& background, const Mat& foreground, + Mat& alphaMap, std::vector& omega_1, std::vector& omega_2, std::vector& omega_3){ + + int width = background.cols; + int height = background.rows; + int npixels = width*height; + + + CV_Assert(omega_1.size() == 0 || omega_1.size() == inputSequence.size()*background.total()); + CV_Assert(omega_2.size() == 0 || omega_2.size() == background.total()); + CV_Assert(omega_3.size() == 0 || omega_3.size() == background.total()); + + float varepsilon = pow(0.001,2); + int deriv_ddepth = CV_32F; + + Mat backgd_dx; + Mat backgd_dy; + Mat obstruc_dx; + Mat obstruc_dy; + + + //compute gradients of current background and occlusion components + Sobel(background, backgd_dx, deriv_ddepth, 1, 0); + Sobel(background, backgd_dy, deriv_ddepth, 0, 1); + Sobel(foreground, obstruc_dx, deriv_ddepth, 1, 0); + Sobel(foreground, obstruc_dy, deriv_ddepth, 0, 1); + + //if weights have not been initialized + if (omega_1.size()==0 && omega_2.size() == 0 && omega_3.size() == 0){ + + //compute derivative denominators (weights) + for(size_t tt=0; ttbackFlowFields[tt]; + Mat obstruc_flow=this->foreFlowFields[tt]; + Mat temp = img-imgWarpFlow(foreground,obstruc_flow)-imgWarpFlow(alphaMap,obstruc_flow).mul(imgWarpFlow(background,back_flow)); + for(size_t i=0; i(i)*temp.at(i)+varepsilon)); + } + } + for(size_t i=0;i(i)*backgd_dx.at(i)+backgd_dy.at(i)*backgd_dy.at(i)+varepsilon)) ; + omega_3.push_back(1/sqrt(obstruc_dx.at(i)*obstruc_dx.at(i)+obstruc_dy.at(i)*obstruc_dy.at(i)+varepsilon)); + } + } + + //if weights have been calculated + else if (omega_1.size() == inputSequence.size()*background.total() && omega_2.size() == background.total() && + omega_3.size() == background.total()){ + + //compute derivative denominators (weights) + for(size_t tt=0; ttbackFlowFields[tt]; + Mat obstruc_flow=this->foreFlowFields[tt]; + Mat temp = img-imgWarpFlow(foreground,obstruc_flow)-imgWarpFlow(alphaMap,obstruc_flow).mul(imgWarpFlow(background,back_flow)); + for(size_t i=0; i(i)*temp.at(i)+varepsilon); + } + } + for(size_t i=0;i(i)*backgd_dx.at(i)+backgd_dy.at(i)*backgd_dy.at(i)+varepsilon); + omega_3[i] = 1/sqrt(obstruc_dx.at(i)*obstruc_dx.at(i)+obstruc_dy.at(i)*obstruc_dy.at(i)+varepsilon); + } + } +} //core function void obstructionFree::removeOcc(const std::vector &srcImgs, Mat &dst, Mat& foreground, Mat &mask, const int obstructionType){ From e9c6f9c2876ee0fe7c170ad3af9d0a2e29fd1089 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Sat, 22 Jul 2017 17:20:26 +0900 Subject: [PATCH 16/17] 1. Fix the conversion between int and size_t. 2. Fix the varepsilon type. --- modules/xphoto/src/obstruction_free.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index dec0e563126..b58a39ac5ac 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -323,7 +323,7 @@ void obstructionFree::motDecomIrlsWeight(const std::vector& inputSequence, CV_Assert(omega_2.size() == 0 || omega_2.size() == background.total()); CV_Assert(omega_3.size() == 0 || omega_3.size() == background.total()); - float varepsilon = pow(0.001,2); + float varepsilon = 0.000001f; int deriv_ddepth = CV_32F; Mat backgd_dx; @@ -347,11 +347,11 @@ void obstructionFree::motDecomIrlsWeight(const std::vector& inputSequence, Mat back_flow=this->backFlowFields[tt]; Mat obstruc_flow=this->foreFlowFields[tt]; Mat temp = img-imgWarpFlow(foreground,obstruc_flow)-imgWarpFlow(alphaMap,obstruc_flow).mul(imgWarpFlow(background,back_flow)); - for(size_t i=0; i(i)*temp.at(i)+varepsilon)); } } - for(size_t i=0;i(i)*backgd_dx.at(i)+backgd_dy.at(i)*backgd_dy.at(i)+varepsilon)) ; omega_3.push_back(1/sqrt(obstruc_dx.at(i)*obstruc_dx.at(i)+obstruc_dy.at(i)*obstruc_dy.at(i)+varepsilon)); } @@ -367,14 +367,15 @@ void obstructionFree::motDecomIrlsWeight(const std::vector& inputSequence, Mat back_flow=this->backFlowFields[tt]; Mat obstruc_flow=this->foreFlowFields[tt]; Mat temp = img-imgWarpFlow(foreground,obstruc_flow)-imgWarpFlow(alphaMap,obstruc_flow).mul(imgWarpFlow(background,back_flow)); - for(size_t i=0; i(i + tt*npixels); omega_1[offset] = 1/sqrt(temp.at(i)*temp.at(i)+varepsilon); } } - for(size_t i=0;i(i)*backgd_dx.at(i)+backgd_dy.at(i)*backgd_dy.at(i)+varepsilon); - omega_3[i] = 1/sqrt(obstruc_dx.at(i)*obstruc_dx.at(i)+obstruc_dy.at(i)*obstruc_dy.at(i)+varepsilon); + for(int i=0;i(i); + omega_2[offset] = 1/sqrt(backgd_dx.at(i)*backgd_dx.at(i)+backgd_dy.at(i)*backgd_dy.at(i)+varepsilon); + omega_3[offset] = 1/sqrt(obstruc_dx.at(i)*obstruc_dx.at(i)+obstruc_dy.at(i)*obstruc_dy.at(i)+varepsilon); } } } From 9a0caa8865fa27ec315bd6ee06700f6d9fc33ad2 Mon Sep 17 00:00:00 2001 From: Binbin Xu Date: Mon, 24 Jul 2017 00:57:21 +0900 Subject: [PATCH 17/17] add D_x^TD_x + D_y^TD_y --- .../opencv2/xphoto/obstruction_free.hpp | 4 ++ modules/xphoto/src/obstruction_free.cpp | 62 +++++++++++++++++++ 2 files changed, 66 insertions(+) diff --git a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp index d0f060b1d40..ff510a2b070 100644 --- a/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp +++ b/modules/xphoto/include/opencv2/xphoto/obstruction_free.hpp @@ -130,6 +130,10 @@ namespace xphoto */ Mat indexToMask(const Mat& indexMat, const int rows, const int cols); + /** @brief Calculate laplacian filters D_x^TD_x + D_y^TD_y + */ + Mat Laplac(const Mat& input); + /*! * @brief Calculate the weights in the alternative motion decomposition step * @param inputSequence Input sequences diff --git a/modules/xphoto/src/obstruction_free.cpp b/modules/xphoto/src/obstruction_free.cpp index b58a39ac5ac..14aab32419d 100644 --- a/modules/xphoto/src/obstruction_free.cpp +++ b/modules/xphoto/src/obstruction_free.cpp @@ -310,6 +310,68 @@ Mat obstructionFree::imgInitDecomOpaq(const std::vector &warpedImgs, Mat& return background; } +//D_x^TD_x + D_y^TD_y +Mat obstructionFree::Laplac(const Mat& input) + { + CV_Assert(input.type() == CV_8U ||input.type() == CV_32F); + + Mat _input; + if (input.type() == CV_8U) input.convertTo(_input, CV_32F); + else _input=input.clone(); + + int width=input.cols; + int height=input.rows; + + _input = _input.reshape(0,1); + Size s=_input.size(); + Mat temp=Mat::zeros(s, CV_32FC1); + Mat _output=Mat::zeros(s, CV_32FC1); + Mat output =Mat::zeros(height, width, CV_32FC1); + + // horizontal filtering + for(int i=0;i(offset)=_input.at(offset+1) - _input.at(offset); + } + + for(int i=0;i(offset) -= temp.at(offset); + if(j>0) + _output.at(offset) += temp.at(offset-1); + } + + temp.release(); + + temp=Mat::zeros(s, CV_32FC1); + + // vertical filtering + for(int i=0;i(offset)=_input.at(offset+width)- _input.at(offset); + } + for(int i=0;i(offset) -= temp.at(offset); + if(i>0) + _output.at(offset) += temp.at(offset-width); + } + + output = _output.reshape(0, input.rows); + //output.convertTo(output, input.type()); + return output; + } + //Calculate the weights in the alternative motion decomposition step void obstructionFree::motDecomIrlsWeight(const std::vector& inputSequence, const Mat& background, const Mat& foreground, Mat& alphaMap, std::vector& omega_1, std::vector& omega_2, std::vector& omega_3){