Skip to content

Commit 2737b56

Browse files
MasterqsxMasterqsx
committed
working on algorithm sp
1 parent f344de4 commit 2737b56

File tree

2 files changed

+167
-15
lines changed

2 files changed

+167
-15
lines changed

‎modules/saliency/include/opencv2/saliency/saliencySpecializedClasses.hpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,7 @@ class CV_EXPORTS_W DiscriminantSaliency : public MotionSaliency
315315
unsigned windowSize;
316316
unsigned patchSize;
317317
unsigned temporalSize;
318+
unsigned patchNumber;
318319
public:
319320
struct DT
320321
{
@@ -339,10 +340,11 @@ class CV_EXPORTS_W DiscriminantSaliency : public MotionSaliency
339340
return computeSaliencyImpl( image, saliencyMap );
340341
}
341342
void dynamicTextureEstimator( const Mat, DT& );
343+
void patchGenerator( const std::vector<Mat>& img_sq, unsigned index, unsigned r, unsigned c, Mat& center, Mat& surround, Mat& all );
344+
std::vector<Mat> saliencyMapGenerator( const std::vector<Mat> );
342345
protected:
343346
bool computeSaliencyImpl( InputArray image, OutputArray saliencyMap );
344-
std::vector<Mat> saliencyMapGenerator( const std::vector<Mat> );
345-
double KLdivDT( const Mat, const DT&, const DT& );
347+
double KLdivDT( const DT&, const DT& );
346348
};
347349

348350
/************************************ Specific Objectness Specialized Classes ************************************/

‎modules/saliency/src/DiscriminantSaliency.cpp

Lines changed: 163 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ DiscriminantSaliency::DiscriminantSaliency()
3434
windowSize = 96;
3535
patchSize = 8;
3636
temporalSize = 11;
37+
patchNumber = 16;
38+
CV_Assert( centerSize > patchSize && pow(centerSize - patchSize, 2) >= patchNumber );
3739
CV_Assert( hiddenSpaceDimension <= temporalSize && temporalSize <= (unsigned)imgProcessingSize.width * imgProcessingSize.height );
3840
}
3941

@@ -46,39 +48,187 @@ bool DiscriminantSaliency::computeSaliencyImpl( InputArray image, OutputArray sa
4648

4749
vector<Mat> DiscriminantSaliency::saliencyMapGenerator( const std::vector<Mat> img_sq)
4850
{
51+
CV_Assert( img_sq.size() >= temporalSize );
52+
vector<Mat> saliency_sq;
53+
for ( unsigned i = temporalSize - 1; i < img_sq.size(); i++ )
54+
{
55+
saliency_sq.push_back(Mat(imgProcessingSize, CV_64F, Scalar::all(0.0)));
56+
for ( unsigned r = (centerSize - 1) / 2; r < imgProcessingSize.height - (centerSize - (centerSize - 1) / 2); r++)
57+
{
58+
for ( unsigned c = (centerSize - 1) / 2; c < imgProcessingSize.width - (centerSize - (centerSize - 1) / 2); c++)
59+
{
60+
Mat center, surround, all;
61+
DT para_c0, para_c1, para_w;
62+
patchGenerator(img_sq, i, r, c, center, surround, all);
63+
dynamicTextureEstimator( surround, para_c0 );
64+
dynamicTextureEstimator( center, para_c1 );
65+
dynamicTextureEstimator( all, para_w );
66+
saliency_sq[0].at<double>(r, c) = 0.5 * KLdivDT( para_c0, para_w ) + 0.5 * KLdivDT( para_c1, para_w );
67+
}
68+
}
69+
}
70+
return saliency_sq;
71+
}
72+
73+
void DiscriminantSaliency::patchGenerator( const vector<Mat>& img_sq, unsigned index, unsigned r, unsigned c, Mat& center, Mat& surround, Mat& all )
74+
{
75+
unsigned r1 = max(r - (windowSize - 1) / 2, (unsigned)0), c1 = max(c - (windowSize - 1) / 2, (unsigned)0);
76+
unsigned r2 = min(r1 + windowSize, (unsigned)imgProcessingSize.height), c2 = min(c1 + windowSize, (unsigned)imgProcessingSize.width);
77+
all = Mat(patchSize * patchSize * patchNumber, temporalSize, CV_64F, Scalar::all(0.0));
78+
surround = Mat(patchSize * patchSize * patchNumber, temporalSize, CV_64F, Scalar::all(0.0));
79+
center = Mat(patchSize * patchSize * patchNumber, temporalSize, CV_64F, Scalar::all(0.0));
80+
81+
for ( int i = 0; i < all.size[0]; i++ )
82+
{
83+
srand(0);
84+
unsigned rt = rand() % (r2 - r1) + r1, ct = rand() % (c2 - c1) + c1;
85+
for ( int j = 0; j < all.size[1]; j++ )
86+
{
87+
all.at<double>(i, j) = img_sq[j + 1 + index - temporalSize].at<double>(rt, ct);
88+
}
89+
}
90+
91+
for ( int i = 0; i < center.size[0]; i++ )
92+
{
93+
srand(0);
94+
unsigned rt = rand() % centerSize + r - (centerSize - 1) / 2, ct = rand() % centerSize + c - (centerSize - 1) / 2;
95+
for ( int j = 0; j < center.size[1]; j++ )
96+
{
97+
center.at<double>(i, j) = img_sq[j + 1 + index - temporalSize].at<double>(rt, ct);
98+
}
99+
}
100+
101+
for ( int i = 0; i < surround.size[0]; i++ )
102+
{
103+
srand(0);
104+
unsigned rt = rand() % (r2 - r1) + r1, ct = rand() % (c2 - c1) + c1;
105+
while ((abs(rt - r) < (centerSize / 2)) && (abs(ct - c) < (centerSize / 2)))
106+
{
107+
rt = rand() % (r2 - r1) + r1;
108+
ct = rand() % (c2 - c1) + c1;
109+
}
110+
for ( int j = 0; j < surround.size[1]; j++ )
111+
{
112+
surround.at<double>(i, j) = img_sq[j + 1 + index - temporalSize].at<double>(rt, ct);
113+
}
114+
}
49115

50-
return img_sq;
51116
}
52117

53-
double DiscriminantSaliency::KLdivDT( const Mat img_sq, DT& para_c, DT& para_w )
118+
double DiscriminantSaliency::KLdivDT( const DT& para_c, const DT& para_w )
54119
{
120+
Mat temp1, temp2, Eig;
55121
Mat MU_c = para_c.MU.clone();//1
56122
Mat MU_w = para_w.MU.clone();//1
57123
Mat S_c = para_c.S.clone();//1
58124
Mat S_w = para_w.S.clone();//1
59-
Mat Beta = ((para_w.S.inv()) + Mat::ones(para_w.S.size(), CV_64F) / (para_w.VAR)).inv();//1
125+
Mat Beta = ((para_w.S.inv()) + Mat::eye(para_w.S.size(), CV_64F) / (para_w.VAR)).inv();//1
126+
Mat Beta_c = ((para_c.S.inv()) + Mat::eye(para_c.S.size(), CV_64F) / (para_c.VAR)).inv();//1
60127
Mat Omega = -1 * para_w.Q.inv() * para_w.A;//const
128+
Mat Omega_c = -1 * para_c.Q.inv() * para_c.A;
61129
Mat Theta = (para_w.S.inv()) + (para_w.A.t()) * (para_w.Q.inv()) * (para_w.A);//const
130+
Mat Theta_c = (para_c.S.inv()) + (para_c.A.t()) * (para_c.Q.inv()) * (para_c.A);
131+
Mat Tc = para_w.C.t() * para_c.C;//const
62132
Mat Vc = para_w.C.t() * para_c.C * MU_c - MU_w;//1
133+
double Omegac = trace(para_c.S).val[0] / para_w.VAR + imgProcessingSize.width * imgProcessingSize.height * para_c.VAR / para_w.VAR - para_c.VAR / pow(para_w.VAR, 2) * trace(Beta).val[0];
134+
Mat Psic = 1 / pow(para_w.VAR, 2) * Tc * para_c.S * Tc.t();
63135

136+
Mat U_c = para_c.A * para_c.S;//2
64137
Mat U_w = para_w.A * para_w.S;//2
65-
Mat H = Theta + Mat::ones(Theta.size(), CV_64F) / (para_w.VAR) - Omega.t() * Beta * Omega;//2
66138
Mat G = -1 * Beta * Omega;//2
139+
Mat G_c = -1 * Beta_c * Omega_c;
140+
Mat H = Theta + Mat::eye(Theta.size(), CV_64F) / (para_w.VAR) - Omega.t() * Beta * Omega;//2
141+
Mat H_c = Theta_c + Mat::eye(Theta_c.size(), CV_64F) / (para_c.VAR) - Omega_c.t() * Beta_c * Omega_c;
142+
143+
S_w = para_w.A * S_w * para_w.A.t() + para_w.Q;//2
67144
MU_c *= para_c.A;//2
68145
MU_w *= para_w.A;//2
69-
S_c = para_c.A * S_c * para_c.A.t() + para_c.Q;
70-
S_w = para_w.A * S_w * para_w.A.t() + para_w.Q;
71-
/*Beta =
146+
Mat Zc = 1 / para_w.VAR * para_w.C * U_w * (Mat::eye(Beta.size(), CV_64F) - Beta / para_w.VAR) * Vc - para_c.C * MU_c + para_w.C * MU_w;//2
147+
148+
eigen((S_w - 1 / para_w.VAR * U_w * (Mat::eye(Beta.size(), CV_64F) - Beta / para_w.VAR) * U_w.t()), Eig);
149+
Eig /= para_w.VAR;
150+
Eig += Mat::ones(Eig.size(), CV_64F);
151+
log(Eig, Eig);
152+
double det_w = imgProcessingSize.width * imgProcessingSize.height * log(para_w.VAR) + sum(Eig).val[0];
72153

154+
eigen((S_c - 1 / para_c.VAR * U_c * (Mat::eye(Beta.size(), CV_64F) - Beta_c / para_c.VAR) * U_c.t()), Eig);
155+
Eig /= para_c.VAR;
156+
Eig += Mat::ones(Eig.size(), CV_64F);
157+
log(Eig, Eig);
158+
double det_c = imgProcessingSize.width * imgProcessingSize.height * log(para_c.VAR) + sum(Eig).val[0];
73159

74-
for ( unsigned i = 1; i < img_sq.size[1]; i++)
160+
Mat Gama = (S_w - 1 / para_w.VAR * U_w * (Mat::eye(Beta.size(), CV_64F) - Beta / para_w.VAR) * U_w.t()).inv() + Mat::eye(S_w.size(), CV_64F) / para_w.VAR;//2
161+
temp1 = (1 / para_w.VAR / para_w.VAR * Zc.t() * para_w.C * Gama.inv() * para_w.C.t() * Zc);//2
162+
163+
double update_term = pow(norm(Zc), 2) / para_w.VAR - temp1.at<double>(0, 0);
164+
Mat Xic = 1 / pow(para_w.VAR, 2) * para_c.A * para_c.S * Tc.t();
165+
166+
S_c = para_c.A * S_c * para_c.A.t() + para_c.Q;//2
167+
168+
Omegac += 1 / para_w.VAR * trace(S_c).val[0] + imgProcessingSize.width * imgProcessingSize.height * para_c.VAR / para_w.VAR - para_c.VAR / pow(para_w.VAR, 2) * trace(H.inv()).val[0] - para_c.VAR / pow(para_w.VAR, 2) * trace(H.inv() * G.t() * G).val[0];
169+
170+
hconcat(Psic, Xic.t() * Tc.t(), temp1);
171+
hconcat(Tc * Xic, 1 / pow(para_w.VAR, 2) * Tc * S_c * Tc.t(), temp2);
172+
vconcat(temp1, temp2, Psic);//Psic 2
173+
174+
hconcat(H.inv(), H.inv() * G.t(), temp1);
175+
hconcat(G * H.inv(), Beta + G * H.inv() * G.t(), temp2);
176+
vconcat(temp1, temp2, Beta);//Beta 2
177+
178+
hconcat(H_c.inv(), H_c.inv() * G_c.t(), temp1);
179+
hconcat(G_c * H_c.inv(), Beta_c + G_c * H_c.inv() * G_c.t(), temp2);
180+
vconcat(temp1, temp2, Beta_c);
181+
182+
vconcat(Vc, para_w.C.t() * para_c.C * MU_c - MU_w, Vc);//2
183+
184+
for ( unsigned i = 2; i < temporalSize; i++)
75185
{
186+
hconcat(para_c.A * U_c, para_c.A * S_c, U_c);
187+
hconcat(para_w.A * U_w, para_w.A * S_w, U_w);
188+
hconcat(-1 * H.inv() * Omega, -1 * G * H.inv() * Omega, G);
189+
hconcat(-1 * H_c.inv() * Omega_c, -1 * G_c * H_c.inv() * Omega_c, G_c);
190+
H = Theta + Mat::eye(Theta.size(), CV_64F) / para_w.VAR - Omega.t() * H.inv() * Omega;
191+
H_c = Theta_c + Mat::eye(Theta_c.size(), CV_64F) / para_c.VAR - Omega_c.t() * H_c.inv() * Omega_c;
76192

193+
S_w = para_w.A * S_w * para_w.A.t() + para_w.Q;
194+
MU_c *= para_c.A;
195+
MU_w *= para_w.A;
196+
Zc = 1 / para_w.VAR * para_w.C * U_w * (Mat::eye(Beta.size(), CV_64F) - Beta / para_w.VAR) * Vc - para_c.C * MU_c + para_w.C * MU_w;
197+
198+
eigen((S_w - 1 / para_w.VAR * U_w * (Mat::eye(Beta.size(), CV_64F) - Beta / para_w.VAR) * U_w.t()), Eig);
199+
Eig /= para_w.VAR;
200+
Eig += Mat::ones(Eig.size(), CV_64F);
201+
log(Eig, Eig);
202+
det_w += imgProcessingSize.width * imgProcessingSize.height * log(para_w.VAR) + sum(Eig).val[0];
203+
204+
eigen((S_c - 1 / para_c.VAR * U_c * (Mat::eye(Beta.size(), CV_64F) - Beta_c / para_c.VAR) * U_c.t()), Eig);
205+
Eig /= para_c.VAR;
206+
Eig += Mat::ones(Eig.size(), CV_64F);
207+
log(Eig, Eig);
208+
det_c += imgProcessingSize.width * imgProcessingSize.height * log(para_c.VAR) + sum(Eig).val[0];
209+
210+
Gama = (S_w - 1 / para_w.VAR * U_w * (Mat::eye(Beta.size(), CV_64F) - Beta / para_w.VAR) * U_w.t()).inv() + Mat::eye(S_w.size(), CV_64F) / para_w.VAR;
211+
temp1 = pow(norm(Zc), 2) / para_w.VAR - 1 / para_w.VAR / para_w.VAR * Zc.t() * para_w.C * Gama.inv() * para_w.C.t() * Zc;
212+
update_term += temp1.at<double>(0, 0);
213+
hconcat(1/pow(para_w.VAR, 2) * para_c.A * Xic, 1/pow(para_w.VAR, 2) * para_c.A * S_c * Tc.t(), Xic);
214+
S_c = para_c.A * S_c * para_c.A.t() + para_c.Q;
215+
Omegac += 1 / para_w.VAR * trace(S_c).val[0] + imgProcessingSize.width * imgProcessingSize.height * para_c.VAR / para_w.VAR - para_c.VAR / pow(para_w.VAR, 2) * trace(H.inv()).val[0] - para_c.VAR / pow(para_w.VAR, 2) * trace(H.inv() * G.t() * G).val[0];
216+
217+
hconcat(Psic, Xic.t() * Tc.t(), temp1);
218+
hconcat(Tc * Xic, 1 / pow(para_w.VAR, 2) * Tc * S_c * Tc.t(), temp2);
219+
vconcat(temp1, temp2, Psic);
220+
221+
hconcat(H.inv(), H.inv() * G.t(), temp1);
222+
hconcat(G * H.inv(), Beta + G * H.inv() * G.t(), temp2);
223+
vconcat(temp1, temp2, Beta);
224+
225+
hconcat(H_c.inv(), H_c.inv() * G_c.t(), temp1);
226+
hconcat(G_c * H_c.inv(), Beta_c + G_c * H_c.inv() * G_c.t(), temp2);
227+
vconcat(temp1, temp2, Beta_c);
228+
229+
vconcat(Vc, para_w.C.t() * para_c.C * MU_c - MU_w, Vc);
77230
}
78-
Mat Zc
79-
80-
Mat Gama*/
81-
return 0;
231+
return det_w - det_c + update_term + Omegac - trace(Beta * Psic).val[0] - imgProcessingSize.width * imgProcessingSize.height * temporalSize;
82232
}
83233

84234
void DiscriminantSaliency::dynamicTextureEstimator( const Mat img_sq, DT& para )
@@ -117,7 +267,7 @@ void DiscriminantSaliency::dynamicTextureEstimator( const Mat img_sq, DT& para )
117267
B = sw.u * Mat::diag(sw.w) / sqrt(tau - 1);
118268
para.R = (B * B.t());
119269
para.R = para.R.clone();
120-
para.VAR = (trace(para.R) / para.R.size[0])[0];
270+
para.VAR = (trace(para.R).val[0] / para.R.size[0]);
121271

122272
para.MU = Mat( hiddenSpaceDimension, 1, CV_64F, Scalar::all(0.0) );
123273
for (unsigned i = 0; i < tau; i++ )

0 commit comments

Comments
 (0)