supported HF-Net

Author: zihaomu

modules/rgbd/samples/large_kinfu_LCD.cpp

@@ -58,11 +58,12 @@ static const char* keys = {
     " in coarse mode points and normals are displayed }"
     "{idle   | | Do not run LargeKinfu, just display depth frames }"
     "{record | | Write depth frames to specified file list (the same format as for the 'depth' key) }"
-    "{modelBin | | Path to a binary .caffemodel file contains trained network which can be download at URL=http://udel.edu/~nmerrill/calc.tar.gz.}"
-    "{modelTxt | | Path to a .prototxt file contains the model definition of trained network.}"
-    "{width  | 160 | Preprocess input image by resizing to a specific width. It should be multiple by 32. }"
-    "{height | 120 | Preprocess input image by resizing to a specific height. It should be multiple by 32. }"
-    "{backend | 0 | Choose one of computation backends: "
+    "{modelBin | | Path to a binary .bin file contains trained network which can be download at URL=https://1drv.ms/u/s!ApQBoiZSe8Evgolqw23hI8D7lP9mKw?e=ywHAc5}"
+    "{modelTxt | | Path to a .xml file contains the model definition of trained network.}"
+    "{width  | 640 | Preprocess input image by resizing to a specific width. }"
+    "{height | 480 | Preprocess input image by resizing to a specific height. It should be multiple by 32. }"
+    "{backend | 2 | At current stage only openvino available, and other backend will be supported soon."
+                    " Choose one of computation backends: "
                     "0: automatically (by default), "
                     "1: Halide language (http://halide-lang.org/), "
                     "2: Intel's Deep Learning Inference Engine (https://software.intel.com/openvino-toolkit), "
@@ -80,7 +81,8 @@ static const std::string message =
     "\nto demonstrate Submap based large environment reconstruction"
     "\nThis module uses the newer hashtable based TSDFVolume (relatively fast) for larger "
     "reconstructions by default\n"
-    "\n The DNN model can be downdload at URL=http://udel.edu/~nmerrill/calc.tar.gz .\n";
+    "\n The used OpenVINO DNN model can be downdload at URL=https://1drv.ms/u/s!ApQBoiZSe8Evgolqw23hI8D7lP9mKw?e=ywHAc5.\n"
+    "\n Make sure that OpenVINO DNN backend is available.\n";
 
 int main(int argc, char** argv)
 {

modules/rgbd/src/large_kinfu.cpp

@@ -323,7 +323,10 @@ bool LargeKinfuImpl<MatType>::updateT(const MatType& _depth, const Mat& _img)
             {
                 // Adding Loop Edge for optimize. If the Edge is duplicate, then skip.
                 if(submapMgr->addEdgeToCurrentSubmap(currentSubmapId, tarSubmapID))
-                    CV_LOG_INFO(NULL, "There is Loop Closure!!!! New edge was added from Submap :"<<currentSubmapId<<" to Submap:"<<tarSubmapID);
+                    CV_LOG_INFO(NULL, "LCD: Find a NEW LOOP! from Submap :"<<currentSubmapId<<" to Submap:"<<tarSubmapID);
+            }else
+            {
+                CV_LOG_INFO(NULL, "LCD: No Loop.");
             }
         }
     }

modules/rgbd/src/loop_closure_detection.cpp

@@ -19,8 +19,12 @@ LoopClosureDetectionImpl::LoopClosureDetectionImpl(const String& _modelBin, cons
         net = makePtr<dnn::Net>(dnn::readNet(_modelBin, _modelTxt));
     }
 
+    //Only HF-Net with OpenVINO backend was supported.
+    // Pre-trained model can be found at https://1drv.ms/u/s!ApQBoiZSe8Evgolqw23hI8D7lP9mKw?e=ywHAc5.
+    //! TODO: HF-Net with OpenCV DNN backend.
     net->setPreferableBackend(_backendId);
     net->setPreferableTarget(_targetId);
+    outNameDNN = net->getUnconnectedOutLayersNames();
 
     KFDataBase = makePtr<KeyFrameDatabase>(maxDatabaseSize);
 }
@@ -115,7 +119,9 @@ bool LoopClosureDetectionImpl::loopCheck(int& tarSubmapID)
 
     // find target submap ID
     if(bestLoopFrame->submapID == -1 || bestLoopFrame->submapID == currentSubmapID)
+    {
         return false;
+    }
     else
     {
         tarSubmapID = bestLoopFrame->submapID;
@@ -151,7 +157,7 @@ bool LoopClosureDetectionImpl::ORBMather(InputArray feature1, InputArray feature
     }
     if(goodMatches.size() < ORBminMathing)
     {
-        CV_LOG_INFO(NULL, "Currently there are too few ORB matching pairs.");
+        CV_LOG_INFO(NULL, "LOOP CLOSURE: There are too few ORB matching pairs.");
         return false;
     }
     else
@@ -207,15 +213,17 @@ void LoopClosureDetectionImpl::reset()
 
 void LoopClosureDetectionImpl::processFrame(InputArray img, Mat& DNNfeature, std::vector<KeyPoint>& currentKeypoints, Mat& ORBFeature)
 {
+    std::vector<Mat> outMats;
+
     // DNN processing.
     Mat imgBlur, outDNN, outORB;
-    cv::GaussianBlur(img, imgBlur, cv::Size(7, 7), 0);
-    Mat blob = dnn::blobFromImage(imgBlur, 1.0/255.0, inputSize);
+    Mat blob = dnn::blobFromImage(imgBlur, 1.0, inputSize);
+
     net->setInput(blob);
-    net->forward(outDNN);
+    net->forward(outMats, outNameDNN);
 
-    outDNN /= norm(outDNN);
-    DNNfeature = outDNN.clone();
+    outMats[0] /= norm(outMats[0]);
+    DNNfeature = outMats[0].clone();
 
     // ORB process
 #ifdef HAVE_OPENCV_FEATURES2D

modules/rgbd/src/loop_closure_detection.hpp

@@ -53,17 +53,19 @@ class LoopClosureDetectionImpl : public LoopClosureDetection
 #endif
     size_t ORBminMathing = 10;
 
-    // Param: Only for DeepLCD
-    int minDatabaseSize = 5;
+    int minDatabaseSize = 50;
     int maxDatabaseSize = 2000;
 
     int preLoopedKFID = -1;
 
-    double similarityHigh = 0.94;
-    double similarityLow = 0.92;
+    // Param: HF-Net
+    // Github Link: https://github.com/ethz-asl/hfnet
+    std::vector<String> outNameDNN;
+    double similarityHigh = 0.64;
+    double similarityLow = 0.60;
 
 };
 
 }
 }
-#endif
+#endif