Python bindings for Facemark Training Code

modules/face/include/opencv2/face/face_alignment.hpp

@@ -8,7 +8,7 @@
 
 namespace cv{
 namespace face{
-class CV_EXPORTS_W FacemarkKazemi : public Facemark
+class CV_EXPORTS_W FacemarkKazemi : public FacemarkTrain
 {
 public:
     struct CV_EXPORTS Params
@@ -35,25 +35,15 @@ class CV_EXPORTS_W FacemarkKazemi : public Facemark
         unsigned long num_test_splits;
         /// configfile stores the name of the file containing the values of training parameters
         String configfile;
+        /// modelfile stores the name of the file containing the Kazemi model
+        String modelfile;
+        /// faceCascadefile stores the name of the file containing the face cascade model
+        String faceCascadefile;
+        /// scale to which all the images and the landmarks need to be scaled
+        Size scale;
     };
     static Ptr<FacemarkKazemi> create(const FacemarkKazemi::Params &parameters = FacemarkKazemi::Params());
     virtual ~FacemarkKazemi();
-
-    /** @brief This function is used to train the model using gradient boosting to get a cascade of regressors
-    *which can then be used to predict shape.
-    *@param images A vector of type cv::Mat which stores the images which are used in training samples.
-    *@param landmarks A vector of vectors of type cv::Point2f which stores the landmarks detected in a particular image.
-    *@param scale A size of type cv::Size to which all images and landmarks have to be scaled to.
-    *@param configfile A variable of type std::string which stores the name of the file storing parameters for training the model.
-    *@param modelFilename A variable of type std::string which stores the name of the trained model file that has to be saved.
-    *@returns A boolean value. The function returns true if the model is trained properly or false if it is not trained.
-    */
-    virtual bool training(std::vector<Mat>& images, std::vector< std::vector<Point2f> >& landmarks,std::string configfile,Size scale,std::string modelFilename = "face_landmarks.dat")=0;
-
-    /// set the custom face detector
-    virtual bool setFaceDetector(bool(*f)(InputArray , OutputArray, void*), void* userData)=0;
-    /// get faces using the custom detector
-    virtual bool getFaces(InputArray image, OutputArray faces)=0;
 };
 
 }} // namespace

modules/face/include/opencv2/face/facemark.hpp

@@ -76,6 +76,82 @@ class CV_EXPORTS_W Facemark : public virtual Algorithm
     CV_WRAP virtual bool fit( InputArray image,
                               InputArray faces,
                               OutputArrayOfArrays landmarks) = 0;
+
+    /** @brief Add one training sample to the trainer.
+
+    @param image Input image.
+    @param landmarks The ground-truth of facial landmarks points corresponds to the image.
+
+    <B>Example of usage</B>
+    @code
+    String imageFiles = "../data/images_train.txt";
+    String ptsFiles = "../data/points_train.txt";
+    std::vector<String> images_train;
+    std::vector<String> landmarks_train;
+
+    // load the list of dataset: image paths and landmark file paths
+    loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train);
+
+    Mat image;
+    std::vector<Point2f> facial_points;
+    for(size_t i=0;i<images_train.size();i++){
+        image = imread(images_train[i].c_str());
+        loadFacePoints(landmarks_train[i],facial_points);
+        facemark->addTrainingSample(image, facial_points);
+    }
+    @endcode
+
+    The contents in the training files should follows the standard format.
+    Here are examples for the contents in these files.
+    example of content in the images_train.txt
+    @code
+    /home/user/ibug/image_003_1.jpg
+    /home/user/ibug/image_004_1.jpg
+    /home/user/ibug/image_005_1.jpg
+    /home/user/ibug/image_006.jpg
+    @endcode
+
+    example of content in the points_train.txt
+    @code
+    /home/user/ibug/image_003_1.pts
+    /home/user/ibug/image_004_1.pts
+    /home/user/ibug/image_005_1.pts
+    /home/user/ibug/image_006.pts
+    @endcode
+
+    */
+    CV_WRAP virtual bool addTrainingSample(InputArray image, std::vector<Point2f> & landmarks)=0;
+
+    /** @brief Set parameters in the Facemark Instance.
+
+    @param face_cascade_name Path to the face cascade model
+    @param facemark_model_name Path to the facemark model
+    @param config_file_path Path to the Config file (only for FacemarkKazemi)
+    @param scale vector of floats represent scale (only for FacemarkAAM, FacemarkKazemi)
+
+    <B>Example of usage</B>
+    @code
+    facemark->setParams(filename,modelfilename,configfile_name,scale)
+    @endcode
+    */
+    CV_WRAP virtual bool setParams(const String& face_cascade_name,const String& facemark_model_name, const String& config_file_path, InputArray scale)=0;
+
+    /** @brief Trains a Facemark algorithm using the given dataset.
+    Before the training process, training samples should be added to the trainer
+    using face::addTrainingSample function.
+
+    <B>Example of usage</B>
+    @code
+    FacemarkLBF::Params params;
+    params.model_filename = "ibug68.model"; // filename to save the trained model
+    Ptr<Facemark> facemark = FacemarkLBF::create(params);
+
+    // add training samples (see Facemark::addTrainingSample)
+
+    facemark->training();
+    @endcode
+    */
+    CV_WRAP virtual void training()=0;
 }; /* Facemark*/
 
 

modules/face/include/opencv2/face/facemarkAAM.hpp

@@ -65,6 +65,7 @@ class CV_EXPORTS_W FacemarkAAM : public FacemarkTrain
         void write(FileStorage& /*fs*/) const;
 
         std::string model_filename;
+        std::string cascade_filename;
         int m;
         int n;
         int n_iter;

modules/face/include/opencv2/face/facemark_train.hpp

@@ -92,8 +92,8 @@ loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train);
 */
 CV_EXPORTS_W bool loadDatasetList(String imageList,
                                   String annotationList,
-                                  std::vector<String> & images,
-                                  std::vector<String> & annotations);
+                                  CV_OUT std::vector<String> & images,
+                                  CV_OUT std::vector<String> & annotations);
 
 /** @brief A utility to load facial landmark dataset from a single file.
 
@@ -181,8 +181,8 @@ CV_EXPORTS_W bool loadTrainingData( String imageList, String groundTruth,
 * @param trainimages A vector of type cv::String which stores the name of images whose landmarks are tracked
 * @returns A boolean value. It returns true when it reads the data successfully and false otherwise
 */
-CV_EXPORTS_W bool loadTrainingData(std::vector<String> filename,std::vector< std::vector<Point2f> >
-                          &trainlandmarks,std::vector<String> & trainimages);
+CV_EXPORTS_W bool loadTrainingData(std::vector<String> filename, CV_OUT std::vector< std::vector<Point2f> >
+                          &trainlandmarks, CV_OUT std::vector<String> & trainimages);
 
 /** @brief A utility to load facial landmark information from a given file.
 
@@ -262,71 +262,6 @@ The typical pipeline for facemark detection is listed as follows:
 class CV_EXPORTS_W FacemarkTrain : public Facemark
 {
 public:
-    /** @brief Add one training sample to the trainer.
-
-    @param image Input image.
-    @param landmarks The ground-truth of facial landmarks points corresponds to the image.
-
-    <B>Example of usage</B>
-    @code
-    String imageFiles = "../data/images_train.txt";
-    String ptsFiles = "../data/points_train.txt";
-    std::vector<String> images_train;
-    std::vector<String> landmarks_train;
-
-    // load the list of dataset: image paths and landmark file paths
-    loadDatasetList(imageFiles,ptsFiles,images_train,landmarks_train);
-
-    Mat image;
-    std::vector<Point2f> facial_points;
-    for(size_t i=0;i<images_train.size();i++){
-        image = imread(images_train[i].c_str());
-        loadFacePoints(landmarks_train[i],facial_points);
-        facemark->addTrainingSample(image, facial_points);
-    }
-    @endcode
-
-    The contents in the training files should follows the standard format.
-    Here are examples for the contents in these files.
-    example of content in the images_train.txt
-    @code
-    /home/user/ibug/image_003_1.jpg
-    /home/user/ibug/image_004_1.jpg
-    /home/user/ibug/image_005_1.jpg
-    /home/user/ibug/image_006.jpg
-    @endcode
-
-    example of content in the points_train.txt
-    @code
-    /home/user/ibug/image_003_1.pts
-    /home/user/ibug/image_004_1.pts
-    /home/user/ibug/image_005_1.pts
-    /home/user/ibug/image_006.pts
-    @endcode
-
-    */
-    virtual bool addTrainingSample(InputArray image, InputArray landmarks)=0;
-
-    /** @brief Trains a Facemark algorithm using the given dataset.
-    Before the training process, training samples should be added to the trainer
-    using face::addTrainingSample function.
-
-    @param parameters Optional extra parameters (algorithm dependent).
-
-    <B>Example of usage</B>
-    @code
-    FacemarkLBF::Params params;
-    params.model_filename = "ibug68.model"; // filename to save the trained model
-    Ptr<Facemark> facemark = FacemarkLBF::create(params);
-
-    // add training samples (see Facemark::addTrainingSample)
-
-    facemark->training();
-    @endcode
-    */
-
-    virtual void training(void* parameters=0)=0;
-
     /** @brief Set a user defined face detector for the Facemark algorithm.
     @param detector The user defined face detector function
     @param userData Detector parameters

modules/face/samples/sample_train_landmark_detector.cpp

@@ -89,15 +89,19 @@ int main(int argc,char** argv){
     face_cascade.load(cascade_name);
     FacemarkKazemi::Params params;
     params.configfile = configfile_name;
+    params.faceCascadefile = cascade_name;
+    params.modelfile = modelfile_name;
+    params.scale = scale;
     Ptr<FacemarkKazemi> facemark = FacemarkKazemi::create(params);
+    // std::vector<float> scale(2, 460);
+    // facemark->setParams(cascade_name,modelfile_name,configfile_name,scale);
     facemark->setFaceDetector((FN_FaceDetector)myDetector, &face_cascade);
     //create a vector to store image names
     vector<String> imagenames;
     //create object to get landmarks
-    vector< vector<Point2f> > trainlandmarks,Trainlandmarks;
+    vector< vector<Point2f> > trainlandmarks;
     //gets landmarks and corresponding image names in both the vectors
     //vector to store images
-    vector<Mat> trainimages;
     loadTrainingData(filenames,trainlandmarks,imagenames);
     for(unsigned long i=0;i<300;i++){
         string imgname = imagenames[i].substr(0, imagenames[i].size()-1);
@@ -107,11 +111,10 @@ int main(int argc,char** argv){
             cerr<<string("Image "+img+" not found\n.")<<endl;
             continue;
         }
-        trainimages.push_back(src);
-        Trainlandmarks.push_back(trainlandmarks[i]);
+        facemark->addTrainingSample(src, trainlandmarks[i]);
     }
     cout<<"Got data"<<endl;
-    facemark->training(trainimages,Trainlandmarks,configfile_name,scale,modelfile_name);
+    facemark->training();
     cout<<"Training complete"<<endl;
     return 0;
-}
+}

modules/face/samples/sample_train_landmark_detector2.cpp

@@ -106,16 +106,18 @@ int main(int argc,char** argv){
     face_cascade.load(cascade_name);
     FacemarkKazemi::Params params;
     params.configfile = configfile_name;
+    params.faceCascadefile = cascade_name;
+    params.modelfile = modelfile_name;
+    params.scale = scale;
     Ptr<FacemarkKazemi> facemark = FacemarkKazemi::create(params);
+    // std::vector<float> scale(2, 460);
+    // facemark->setParams(cascade_name,modelfile_name,configfile_name,scale);
     facemark->setFaceDetector((FN_FaceDetector)myDetector, &face_cascade);
 
     std::vector<String> images;
     std::vector<std::vector<Point2f> > facePoints;
     loadTrainingData(imagesList, annotations, images, facePoints, 0.0);
     //gets landmarks and corresponding image names in both the vectors
-    vector<Mat> Trainimages;
-    std::vector<std::vector<Point2f> > Trainlandmarks;
-    //vector to store images
     Mat src;
     for(unsigned long i=0;i<images.size();i++){
         src = imread(images[i]);
@@ -124,11 +126,10 @@ int main(int argc,char** argv){
             cerr<<string("Image not found\n.Aborting...")<<endl;
             continue;
         }
-        Trainimages.push_back(src);
-        Trainlandmarks.push_back(facePoints[i]);
+        facemark->addTrainingSample(src, facePoints[i]);
     }
     cout<<"Got data"<<endl;
-    facemark->training(Trainimages,Trainlandmarks,configfile_name,scale,modelfile_name);
+    facemark->training();
     cout<<"Training complete"<<endl;
     return 0;
 }

modules/face/src/face_alignment.cpp

@@ -27,9 +27,77 @@ bool FacemarkKazemiImpl::setFaceDetector(FN_FaceDetector f, void* userData){
 }
 bool FacemarkKazemiImpl::getFaces(InputArray image, OutputArray faces)
 {
-    CV_Assert(faceDetector);
+    if (!faceDetector)
+    {
+        std::vector<Rect> faces_;
+        defaultFaceDetector(image.getMat(), faces_);
+        Mat(faces_).copyTo(faces);
+        return true;
+    }
     return faceDetector(image, faces, faceDetectorData);
 }
+bool FacemarkKazemiImpl::defaultFaceDetector(const Mat& image, std::vector<Rect>& faces){
+    Mat gray;
+
+    faces.clear();
+
+    if (image.channels() > 1)
+    {
+        cvtColor(image, gray, COLOR_BGR2GRAY);
+    }
+    else
+    {
+        gray = image;
+    }
+
+    equalizeHist(gray, gray);
+
+    if (face_cascade.empty())
+    {
+        { /* check the cascade classifier file */
+            std::ifstream infile;
+            infile.open(params.faceCascadefile.c_str(), std::ios::in);
+            if (!infile)
+                CV_Error_(Error::StsBadArg, ("The cascade classifier model is not found: %s", params.faceCascadefile.c_str()));
+        }
+        face_cascade.load(params.faceCascadefile.c_str());
+        CV_Assert(!face_cascade.empty());
+    }
+    face_cascade.detectMultiScale(gray, faces, 1.05, 2, CASCADE_SCALE_IMAGE, Size(30, 30) );
+    return true;
+}
+bool FacemarkKazemiImpl::getData(void * items){
+    CV_UNUSED(items);
+    return false;
+}
+bool FacemarkKazemiImpl::addTrainingSample(InputArray image, std::vector<Point2f> & landmarks){
+  std::vector<Point2f> & _landmarks = landmarks;
+  training_images.push_back(image.getMat());
+  training_facePoints.push_back(_landmarks);
+  return true;
+}
+bool FacemarkKazemiImpl::setParams(const String& face_cascade_name,const String& facemark_model_name, const String& config_file_path, InputArray scale){
+  if(face_cascade_name.empty() && facemark_model_name.empty() && config_file_path.empty() && scale.empty())
+  {
+    CV_Error_(Error::StsBadArg, ("face cascade name, facemark model name, config file and scale all are empty"));
+  }
+  if(!face_cascade_name.empty())
+      params.faceCascadefile = face_cascade_name;
+  if(!facemark_model_name.empty())
+      params.modelfile = facemark_model_name;
+  if(!config_file_path.empty())
+      params.configfile = config_file_path;
+
+  Mat scale_mat = scale.getMat();
+  std::vector<int> _scale = scale_mat.reshape(1, scale_mat.rows);
+  if(_scale.size() != 2){
+    CV_Error(Error::StsBadArg, "Please set the scale argument properly");
+    return false;
+  }
+  params.scale = Size(_scale[0], _scale[1]);
+  CV_UNUSED(config_file_path);
+  return true;
+}
 FacemarkKazemiImpl::FacemarkKazemiImpl(const FacemarkKazemi::Params& parameters) :
     faceDetector(NULL),
     faceDetectorData(NULL)
@@ -45,6 +113,9 @@ FacemarkKazemi::Params::Params(){
     //These variables are used for training data
     //These are initialised as described in the research paper
     //referenced above
+    configfile = "";
+    modelfile = "";
+    faceCascadefile = "";
     cascade_depth = 15;
     tree_depth = 5;
     num_trees_per_cascade_level = 500;