Skip to content

Commit 92f07ad

Browse files
MasterqsxMasterqsx
committed
roughly finish BD algorithm
1 parent c712d7c commit 92f07ad

File tree

3 files changed

+283
-1
lines changed

3 files changed

+283
-1
lines changed

‎modules/saliency/CMakeLists.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,6 @@ endif()
44

55
set(the_description "Saliency API")
66

7-
ocv_define_module(saliency opencv_imgproc opencv_datasets opencv_features2d opencv_dnn WRAP python)
7+
ocv_define_module(saliency opencv_imgproc opencv_datasets opencv_features2d opencv_dnn opencv_ximgproc WRAP python)
88

99
ocv_warnings_disable(CMAKE_CXX_FLAGS -Woverloaded-virtual)

‎modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -196,6 +196,40 @@ class CV_EXPORTS_W DeepGaze1 : public StaticSaliency
196196
std::vector<unsigned> fixationLoc( Mat, Size );
197197
};
198198

199+
/** @brief the Deep Gaze 1 Saliency approach from
200+
201+
This method use the convolution layers of the pretrained AlexNet, linear combination, center bias and softmax to generate saliency map
202+
*/
203+
class CV_EXPORTS_W BackgroundContrast : public StaticSaliency
204+
{
205+
//private:
206+
207+
public:
208+
BackgroundContrast();
209+
virtual ~BackgroundContrast();
210+
CV_WRAP static Ptr<BackgroundContrast> create()
211+
{
212+
return makePtr<BackgroundContrast>();
213+
}
214+
CV_WRAP bool computeSaliency( InputArray image, OutputArray saliencyMap )
215+
{
216+
if( image.empty() )
217+
return false;
218+
return computeSaliencyImpl( image, saliencyMap );
219+
}
220+
Mat saliencyMapGenerator( const Mat );
221+
void saliencyMapVisualize( InputArray _saliencyMap );
222+
protected:
223+
bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap );
224+
void superpixelSplit( const Mat, Mat&, Mat& );
225+
std::vector<unsigned> getBndPatchIds( const Mat, int = 8);
226+
void getColorPosDis( const Mat, const Mat, Mat&, Mat&, int );
227+
void boundaryConnectivity( const Mat, const Mat, Mat&, std::vector<unsigned>, double = 3.0, double = 7.0 );
228+
void getWeightedContrast( const Mat, const Mat, const Mat, Mat& );
229+
};
230+
231+
232+
199233

200234

201235
/************************************ Specific Motion Saliency Specialized Classes ************************************/

‎modules/saliency/src/BackgroundContrast.cpp

Lines changed: 248 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,248 @@
1+
// This file is part of OpenCV project.
2+
// It is subject to the license terms in the LICENSE file found in the top-level directory
3+
// of this distribution and at http://opencv.org/license.html.
4+
5+
#include <opencv2/imgproc.hpp>
6+
#include <opencv2/highgui.hpp>
7+
#include <opencv2/core.hpp>
8+
#include <opencv2/ximgproc.hpp>
9+
#include <opencv2/imgcodecs.hpp>
10+
#include <opencv2/core/utility.hpp>
11+
#include "precomp.hpp"
12+
#include <vector>
13+
#include <functional>
14+
#include <cmath>
15+
#include <string>
16+
#include <iostream>
17+
#include <cstdlib>
18+
#include <algorithm>
19+
#include <utility>
20+
#include <numeric>
21+
#include <fstream>
22+
#include <set>
23+
24+
25+
26+
using namespace std;
27+
using namespace cv;
28+
using namespace cv::saliency;
29+
using namespace cv::ximgproc;
30+
31+
namespace cv
32+
{
33+
namespace saliency
34+
{
35+
36+
BackgroundContrast::BackgroundContrast()
37+
{
38+
39+
}
40+
BackgroundContrast::~BackgroundContrast(){};
41+
42+
Mat BackgroundContrast::saliencyMapGenerator( const Mat img )
43+
{
44+
Mat idxImg, adjcMatrix, colDistM, posDistM, bdProb, wCtr, saliency;
45+
superpixelSplit(img, idxImg, adjcMatrix);
46+
vector<unsigned> bdIds = getBndPatchIds(idxImg);
47+
for (unsigned i = 0; i < bdIds.size(); i++)
48+
{
49+
for (unsigned j = 0; j < bdIds.size(); j++)
50+
{
51+
adjcMatrix.at<uchar>(i, j) = 1;
52+
}
53+
}
54+
getColorPosDis(img, idxImg, colDistM, posDistM, adjcMatrix.size[0]);
55+
boundaryConnectivity(adjcMatrix, colDistM, bdProb, bdIds);
56+
getWeightedContrast(colDistM, posDistM, bdProb, wCtr);
57+
saliency = Mat(img.size[0], img.size[1], CV_64F, Scalar::all(0.0));
58+
for (int i = 0; i < img.size[0]; i++)
59+
{
60+
for (int j = 0; j < img.size[1]; j++)
61+
{
62+
saliency.at<double>(i, j) = wCtr.at<double>(idxImg.at<unsigned>(i, j), 0);
63+
}
64+
}
65+
return saliency;
66+
}
67+
68+
bool BackgroundContrast::computeSaliencyImpl( InputArray image, OutputArray saliencyMap )
69+
{
70+
return true;
71+
}
72+
73+
void BackgroundContrast::superpixelSplit( const Mat img, Mat& idxImg, Mat& adjcMatrix)
74+
{
75+
Ptr<SuperpixelSEEDS> seeds;
76+
seeds = createSuperpixelSEEDS( img.size().width, img.size().height, img.channels(), min(img.size().width * img.size().height / 600, 300), 4, 2, 5, false);
77+
seeds->iterate( img, 4 );
78+
Mat mask;
79+
adjcMatrix = Mat::eye( seeds->getNumberOfSuperpixels(), seeds->getNumberOfSuperpixels(), CV_8U );
80+
seeds->getLabels(idxImg);
81+
seeds->getLabelContourMask(mask, true);
82+
for ( int i = 0; i < mask.size[0]; i++ )
83+
{
84+
for (int j = 0; j < mask.size[1]; j++ )
85+
{
86+
if (mask.at<uchar>(i, j) != 0)
87+
{
88+
if ( idxImg.at<unsigned>(i, j) != idxImg.at<unsigned>(max(i - 1, 0), j) )
89+
{
90+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i, j), idxImg.at<unsigned>(i - 1, j)) = 2;
91+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i - 1, j), idxImg.at<unsigned>(i, j)) = 2;
92+
}
93+
if ( idxImg.at<unsigned>(i, j) != idxImg.at<unsigned>(min(i + 1, mask.size[0] - 1), j) )
94+
{
95+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i, j), idxImg.at<unsigned>(i + 1, j)) = 2;
96+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i + 1, j), idxImg.at<unsigned>(i, j)) = 2;
97+
}
98+
if ( idxImg.at<unsigned>(i, j) != idxImg.at<unsigned>(i, max(j - 1, 0)) )
99+
{
100+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i, j), idxImg.at<unsigned>(i, j - 1)) = 2;
101+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i, j - 1), idxImg.at<unsigned>(i, j)) = 2;
102+
}
103+
if ( idxImg.at<unsigned>(i, j) != idxImg.at<unsigned>(i, min(j + 1, mask.size[1] - 1)) )
104+
{
105+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i, j), idxImg.at<unsigned>(i, j + 1)) = 2;
106+
adjcMatrix.at<uchar>(idxImg.at<unsigned>(i, j + 1), idxImg.at<unsigned>(i, j)) = 2;
107+
}
108+
}
109+
}
110+
}
111+
}
112+
113+
vector<unsigned> BackgroundContrast::getBndPatchIds( const Mat idxImg, int thickness )
114+
{
115+
set<unsigned> BndPatchIds;
116+
CV_Assert(idxImg.size[0] > 2 * thickness && idxImg.size[1] > 2 * thickness);
117+
for ( int i = 0; i < idxImg.size[0]; i++)
118+
{
119+
for (int j = 0; j < idxImg.size[1]; j++)
120+
{
121+
if ( ((i >= 0 && i < thickness) || (i >= idxImg.size[0] - thickness && i < idxImg.size[0])) || ((j >= 0 && j < thickness) || (j >= idxImg.size[1] - thickness && j < idxImg.size[1])))
122+
{
123+
BndPatchIds.insert(idxImg.at<unsigned>(i, j));
124+
}
125+
}
126+
}
127+
vector<unsigned> res(BndPatchIds.begin(), BndPatchIds.end());
128+
return res;
129+
}
130+
131+
void BackgroundContrast::getColorPosDis( const Mat img, const Mat idxImg, Mat& colDistM, Mat& posDistM, int nOfSP )
132+
{
133+
vector<int> szOfSP = vector<int>(nOfSP, 0);
134+
Mat meanCol = Mat(nOfSP, img.channels(), CV_64F, Scalar::all(0.0));
135+
Mat meanPos = Mat(nOfSP, 2, CV_64F, Scalar::all(0.0));
136+
colDistM = Mat(nOfSP, nOfSP, CV_64F, Scalar::all(0.0));
137+
posDistM = Mat(nOfSP, nOfSP, CV_64F, Scalar::all(0.0));
138+
for (int i = 0; i < img.size[0]; i++ )
139+
{
140+
for (int j = 0; j < img.size[1]; j++ )
141+
{
142+
szOfSP[idxImg.at<unsigned>(i, j)] += 1;
143+
for (int k = 0; k < img.channels(); k++)
144+
{
145+
meanCol.at<double>(idxImg.at<unsigned>(i, j), k) += (double)img.at<Vec3b>(i, j)[k];
146+
}
147+
meanPos.at<double>(idxImg.at<unsigned>(i, j), 0) += (double)i;
148+
meanPos.at<double>(idxImg.at<unsigned>(i, j), 1) += (double)j;
149+
}
150+
}
151+
for (int i = 0; i < nOfSP; i++)
152+
{
153+
meanCol.row(i) /= szOfSP[i];
154+
meanPos.row(i) /= szOfSP[i];
155+
}
156+
meanPos.col(0) /= img.size[0];
157+
meanPos.col(1) /= img.size[1];
158+
159+
for ( int i = 0; i < meanCol.size[1]; i++)
160+
{
161+
Mat temp = ( repeat(meanCol.col(i), 1, nOfSP) - repeat(meanCol.col(i).t(), nOfSP, 1) );
162+
pow(temp, 2, temp);
163+
colDistM += temp;
164+
}
165+
sqrt(colDistM, colDistM);
166+
167+
for ( int i = 0; i < meanPos.size[1]; i++)
168+
{
169+
Mat temp = ( repeat(meanPos.col(i), 1, nOfSP) - repeat(meanPos.col(i).t(), nOfSP, 1) );
170+
pow(temp, 2, temp);
171+
posDistM += temp;
172+
}
173+
sqrt(posDistM, posDistM);
174+
}
175+
176+
void BackgroundContrast::boundaryConnectivity(const Mat adjcMatrix, const Mat colDistM, Mat& bdProb, vector<unsigned> bdIds, double clipVal, double geoSigma)
177+
{
178+
Mat geoDistMatrix = Mat(adjcMatrix.size[0], adjcMatrix.size[1], CV_64F, Scalar::all(0.0));
179+
double ma = 0;
180+
minMaxLoc( colDistM, NULL, &ma );
181+
for ( int i = 0; i < adjcMatrix.size[0]; i++ )
182+
{
183+
for (int j = 0; j < adjcMatrix.size[1]; j++ )
184+
{
185+
if ( adjcMatrix.at<uchar>(i, j) == 0 )
186+
{
187+
geoDistMatrix.at<double>(i, j) = ma * adjcMatrix.size[0]; //fake inf
188+
}
189+
else
190+
{
191+
geoDistMatrix.at<double>(i, j) = colDistM.at<double>(i, j);
192+
}
193+
}
194+
}
195+
for ( int k = 0; k < adjcMatrix.size[0]; k++ )
196+
{
197+
for ( int i = 0; i < adjcMatrix.size[0]; i++ )
198+
{
199+
for (int j = 0; j < adjcMatrix.size[1]; j++ )
200+
{
201+
geoDistMatrix.at<double>(i, j) = min(geoDistMatrix.at<double>(i, j), geoDistMatrix.at<double>(i, k) + geoDistMatrix.at<double>(k, j));
202+
geoDistMatrix.at<double>(j, i) = geoDistMatrix.at<double>(i, j);
203+
}
204+
}
205+
}
206+
pow(geoDistMatrix, 2, geoDistMatrix);
207+
geoDistMatrix /= (-1 * 2 * geoSigma * geoSigma);
208+
exp(geoDistMatrix, geoDistMatrix);
209+
bdProb = Mat(adjcMatrix.size[0], 1, CV_64F, Scalar::all(0.0));
210+
for ( int i = 0; i < adjcMatrix.size[0]; i++ )
211+
{
212+
for ( unsigned j = 0; j < bdIds.size(); j++ )
213+
{
214+
bdProb.at<double>(i, 0) += geoDistMatrix.at<double>(i, bdIds[j]);
215+
}
216+
bdProb.at<double>(i, 0) /= sqrt(sum(geoDistMatrix.row(i)).val[0]);
217+
}
218+
219+
pow(bdProb, 2, bdProb);
220+
bdProb /= (-1 * 2);
221+
exp(bdProb, bdProb);
222+
bdProb = 1 - bdProb;
223+
}
224+
225+
void BackgroundContrast::getWeightedContrast( const Mat colDistM, const Mat posDistM, const Mat bgProb, Mat& wCtr )
226+
{
227+
wCtr = posDistM.clone();
228+
pow(wCtr, 2, wCtr);
229+
wCtr /= (-1 * 2 * 0.16);
230+
exp(wCtr, wCtr);
231+
multiply(colDistM, wCtr, wCtr);
232+
wCtr *= bgProb;
233+
}
234+
235+
236+
void BackgroundContrast::saliencyMapVisualize( InputArray _saliencyMap )
237+
{
238+
Mat saliency = _saliencyMap.getMat().clone();
239+
double mi = 0, ma = 0;
240+
minMaxLoc( saliency, &mi, &ma );
241+
saliency -= mi;
242+
saliency /= ( ma - mi );
243+
imshow( "saliencyVisual", saliency );
244+
waitKey( 0 );
245+
}
246+
247+
}
248+
}

0 commit comments

Comments
 (0)