[GSoC 2017] DeepGaze1 + Discriminant Saliency + BackgroundContrast

modules/cnn_3dobj/testdata/cv/deploy.prototxt

@@ -0,0 +1,149 @@
+name: "CaffeNet"
+input: "data"
+input_dim: 10
+input_dim: 3
+input_dim: 227
+input_dim: 227
+layer {
+  name: "conv1"
+  type: "Convolution"
+  bottom: "data"
+  top: "conv1"
+  convolution_param {
+    num_output: 96
+    kernel_size: 11
+    stride: 4
+  }
+}
+layer {
+  name: "relu1"
+  type: "ReLU"
+  bottom: "conv1"
+  top: "conv1"
+}
+layer {
+  name: "pool1"
+  type: "Pooling"
+  bottom: "conv1"
+  top: "pool1"
+  pooling_param {
+    pool: MAX
+    kernel_size: 3
+    stride: 2
+  }
+}
+layer {
+  name: "norm1"
+  type: "LRN"
+  bottom: "pool1"
+  top: "norm1"
+  lrn_param {
+    local_size: 5
+    alpha: 0.0001
+    beta: 0.75
+  }
+}
+layer {
+  name: "conv2"
+  type: "Convolution"
+  bottom: "norm1"
+  top: "conv2"
+  convolution_param {
+    num_output: 256
+    pad: 2
+    kernel_size: 5
+    group: 2
+  }
+}
+layer {
+  name: "relu2"
+  type: "ReLU"
+  bottom: "conv2"
+  top: "conv2"
+}
+layer {
+  name: "pool2"
+  type: "Pooling"
+  bottom: "conv2"
+  top: "pool2"
+  pooling_param {
+    pool: MAX
+    kernel_size: 3
+    stride: 2
+  }
+}
+layer {
+  name: "norm2"
+  type: "LRN"
+  bottom: "pool2"
+  top: "norm2"
+  lrn_param {
+    local_size: 5
+    alpha: 0.0001
+    beta: 0.75
+  }
+}
+layer {
+  name: "conv3"
+  type: "Convolution"
+  bottom: "norm2"
+  top: "conv3"
+  convolution_param {
+    num_output: 384
+    pad: 1
+    kernel_size: 3
+  }
+}
+layer {
+  name: "relu3"
+  type: "ReLU"
+  bottom: "conv3"
+  top: "conv3"
+}
+layer {
+  name: "conv4"
+  type: "Convolution"
+  bottom: "conv3"
+  top: "conv4"
+  convolution_param {
+    num_output: 384
+    pad: 1
+    kernel_size: 3
+    group: 2
+  }
+}
+layer {
+  name: "relu4"
+  type: "ReLU"
+  bottom: "conv4"
+  top: "conv4"
+}
+layer {
+  name: "conv5"
+  type: "Convolution"
+  bottom: "conv4"
+  top: "conv5"
+  convolution_param {
+    num_output: 256
+    pad: 1
+    kernel_size: 3
+    group: 2
+  }
+}
+layer {
+  name: "relu5"
+  type: "ReLU"
+  bottom: "conv5"
+  top: "conv5"
+}
+layer {
+  name: "pool5"
+  type: "Pooling"
+  bottom: "conv5"
+  top: "pool5"
+  pooling_param {
+    pool: MAX
+    kernel_size: 3
+    stride: 2
+  }
+}

modules/datasets/include/opencv2/datasets/saliency_mit1003.hpp

@@ -0,0 +1,47 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#ifndef OPENCV_DATASETS_SALIENCY_MIT1003_HPP
+#define OPENCV_DATASETS_SALIENCY_MIT1003_HPP
+
+#include <string>
+#include <vector>
+
+#include "opencv2/datasets/dataset.hpp"
+
+#include <opencv2/core.hpp>
+#include <opencv2/imgproc.hpp>
+#include <opencv2/highgui.hpp>
+
+namespace cv
+{
+namespace datasets
+{
+
+//! @addtogroup datasets_saliency
+//! @{
+
+struct SALIENCY_mit1003obj : public Object
+{
+    int id;
+    std::string name;
+    Mat img;
+    Mat fixMap;
+    Mat fixPts;
+};
+
+class CV_EXPORTS SALIENCY_mit1003 : public Dataset
+{
+public:
+    virtual void load(const std::string &path) = 0;
+    virtual std::vector<std::vector<Mat> > getDataset() = 0;
+    static Ptr<SALIENCY_mit1003> create();
+};
+
+//! @}
+
+}
+}
+
+#endif

modules/datasets/samples/saliency_mit1003Sample.cpp

@@ -0,0 +1,29 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+
+#include <opencv2/highgui.hpp>
+#include "opencv2/datasets/saliency_mit1003.hpp"
+#include <vector>
+#include <string>
+
+using namespace std;
+using namespace cv;
+using namespace cv::datasets;
+
+int main(int argc, char** argv)
+{
+    if (argc < 2) return 0;
+
+    Ptr<SALIENCY_mit1003> datasetConnector = SALIENCY_mit1003::create();
+    datasetConnector->load(argv[1]);
+    vector<vector<Mat> > dataset(datasetConnector->getDataset()); //dataset[0] is original img, dataset[1] is fixMap, dataset[2] is fixPts
+    //You can use mit1003 dataset do what ever you want
+    for ( unsigned i = 0; i < dataset[0].size(); i++)
+    {
+        imshow("img", dataset[0][i]);
+        waitKey(0);
+    }
+    return 0;
+}

modules/datasets/samples/track_vot.cpp

@@ -96,4 +96,4 @@ int main(int argc, char *argv[])
 
 	getchar();
 	return 0;
-}
+}

modules/datasets/src/saliency_mit1003.cpp

@@ -0,0 +1,83 @@
+// This file is part of OpenCV project.
+// It is subject to the license terms in the LICENSE file found in the top-level directory
+// of this distribution and at http://opencv.org/license.html.
+
+#include "opencv2/datasets/saliency_mit1003.hpp"
+#include "opencv2/datasets/util.hpp"
+
+namespace cv
+{
+namespace datasets
+{
+
+using namespace std;
+
+class SALIENCY_mit1003Imp : public SALIENCY_mit1003
+{
+public:
+    SALIENCY_mit1003Imp() {}
+    virtual ~SALIENCY_mit1003Imp() {}
+
+    virtual void load(const string &path);
+    virtual vector<vector<Mat> > getDataset();
+
+private:
+    void loadDataset(const string &path);
+};
+
+void SALIENCY_mit1003Imp::load(const string &path)
+{
+    loadDataset(path);
+}
+
+void SALIENCY_mit1003Imp::loadDataset(const string &path)
+{
+    train.push_back( vector< Ptr<Object> >() );
+    test.push_back( vector< Ptr<Object> >() );
+    validation.push_back( vector< Ptr<Object> >() );
+
+    string imgPath( path + "/ALLSTIMULI/" );
+    string fixPath( path + "/ALLFIXATIONMAPS/" );
+
+    vector<string> imgNames;
+
+    getDirList( imgPath, imgNames );
+    for ( unsigned i = 0; i < imgNames.size(); i++ )
+    {
+        Ptr<SALIENCY_mit1003obj> curr( new SALIENCY_mit1003obj );
+        curr->name = imgNames[i].substr( 0, imgNames[i].find_first_of('.') );
+        curr->id = i;
+        curr->img = imread( imgPath + curr->name + ".jpeg" );
+        curr->fixMap = imread( fixPath + curr->name + "_fixMap.jpg", 0 );
+        curr->fixPts = imread( fixPath + curr->name + "_fixPts.jpg", 0 );
+        if ( curr->img.empty() || curr->fixMap.empty() || curr->fixPts.empty() ) continue;
+        train.back().push_back(curr);
+        test.back().push_back(curr);
+        validation.back().push_back(curr);
+    }
+
+}
+
+Ptr<SALIENCY_mit1003> SALIENCY_mit1003::create()
+{
+    return Ptr<SALIENCY_mit1003Imp>(new SALIENCY_mit1003Imp);
+}
+
+vector<vector<Mat> > SALIENCY_mit1003Imp::getDataset()
+{
+    vector<vector<Mat> > res = vector<vector<Mat> >( 3, vector<Mat>() );
+    for ( unsigned i = 0; i < train.size() ;i++ )
+    {
+        for ( unsigned j = 0; j < train[i].size() ;j++ )
+        {
+            Ptr<SALIENCY_mit1003obj> curr(static_cast<SALIENCY_mit1003obj *>(train[i][j].get()));
+            res[0].push_back( curr->img );
+            res[1].push_back( curr->fixMap );
+            res[2].push_back( curr->fixPts );
+        }
+    }
+    return res;
+}
+
+}
+}

modules/saliency/CMakeLists.txt

@@ -4,6 +4,6 @@ endif()
 
 set(the_description "Saliency API")
 
-ocv_define_module(saliency opencv_imgproc opencv_features2d WRAP python)
+ocv_define_module(saliency opencv_imgproc opencv_datasets opencv_features2d opencv_dnn opencv_ximgproc WRAP python)
 
 ocv_warnings_disable(CMAKE_CXX_FLAGS -Woverloaded-virtual)

modules/saliency/doc/saliency.bib

@@ -31,3 +31,29 @@ @inproceedings{FGS
   year={2010},
   organization={Elsevier}
 }
+
+@inproceedings{kummerer2014deep,
+  title={Deep gaze i: Boosting saliency prediction with feature maps trained on imagenet},
+  author={K{\"u}mmerer, Matthias and Theis, Lucas and Bethge, Matthias},
+  journal={arXiv preprint arXiv:1411.1045},
+  year={2014}
+}
+
+@inproceedings{mahadevan2010spatiotemporal,
+  title={Spatiotemporal saliency in dynamic scenes},
+  author={Mahadevan, Vijay and Vasconcelos, Nuno},
+  journal={IEEE transactions on pattern analysis and machine intelligence},
+  volume={32},
+  number={1},
+  pages={171--177},
+  year={2010},
+  publisher={IEEE}
+}
+
+@inproceedings{zhu2014saliency,
+  title={Saliency optimization from robust background detection},
+  author={Zhu, Wangjiang and Liang, Shuang and Wei, Yichen and Sun, Jian},
+  booktitle={Proceedings of the IEEE conference on computer vision and pattern recognition},
+  pages={2814--2821},
+  year={2014}
+}