added support for arbitrary resolutioin, created nn_only for fixed 32…

…0x240 size

.gitignore

@@ -17,8 +17,7 @@ Thirdparty/DBoW2/lib/
 Thirdparty/g2o/build/
 Thirdparty/g2o/config.h
 Thirdparty/g2o/lib/
-Vocabulary/ORBvoc.txt
-Vocabulary/voc*.bin
+Vocabulary/*.bin
 build/
 *~
 lib/

GCN2/Network.md

@@ -1,5 +1,7 @@
 # Pytorch model specifications
 
+original models available at: https://drive.google.com/file/d/1MJMroL5-tl0b9__-OiCfxFP9K6X8kvTT/view
+
 ## GCNv2
 
 ```

GCN2/TUM3.yaml

@@ -5,18 +5,18 @@
 #--------------------------------------------------------------------------------------------
 
 # Camera calibration and distortion parameters (OpenCV) 
-Camera.fx: 267.7
-Camera.fy: 269.6
-Camera.cx: 160.05
-Camera.cy: 123.8
+Camera.fx: 535.4
+Camera.fy: 539.2
+Camera.cx: 320.1
+Camera.cy: 247.6
 
 Camera.k1: 0.0
 Camera.k2: 0.0
 Camera.p1: 0.0
 Camera.p2: 0.0
 
-Camera.width: 320
-Camera.height: 240
+Camera.width: 640
+Camera.height: 480
 
 # Camera frames per second 
 Camera.fps: 30.0
@@ -65,5 +65,4 @@ Viewer.CameraLineWidth: 3
 Viewer.ViewpointX: 0
 Viewer.ViewpointY: -0.7
 Viewer.ViewpointZ: -1.8
-Viewer.ViewpointF: 500
-
+Viewer.ViewpointF: 500

GCN2/TUM3_small.yaml

@@ -0,0 +1,69 @@
+%YAML:1.0
+
+#--------------------------------------------------------------------------------------------
+# Camera Parameters. Adjust them!
+#--------------------------------------------------------------------------------------------
+
+# Camera calibration and distortion parameters (OpenCV) 
+Camera.fx: 267.7
+Camera.fy: 269.6
+Camera.cx: 160.05
+Camera.cy: 123.8
+
+Camera.k1: 0.0
+Camera.k2: 0.0
+Camera.p1: 0.0
+Camera.p2: 0.0
+
+Camera.width: 320
+Camera.height: 240
+
+# Camera frames per second 
+Camera.fps: 30.0
+
+# IR projector baseline times fx (aprox.)
+Camera.bf: 40.0
+
+# Color order of the images (0: BGR, 1: RGB. It is ignored if images are grayscale)
+Camera.RGB: 1
+
+# Close/Far threshold. Baseline times.
+ThDepth: 40.0
+
+# Deptmap values factor
+DepthMapFactor: 5000.0
+
+#--------------------------------------------------------------------------------------------
+# ORB Parameters
+#--------------------------------------------------------------------------------------------
+
+# ORB Extractor: Number of features per image
+ORBextractor.nFeatures: 1000
+
+# ORB Extractor: Scale factor between levels in the scale pyramid 	
+ORBextractor.scaleFactor: 1.2
+
+# ORB Extractor: Number of levels in the scale pyramid	
+ORBextractor.nLevels: 8
+
+# ORB Extractor: Fast threshold
+# Image is divided in a grid. At each cell FAST are extracted imposing a minimum response.
+# Firstly we impose iniThFAST. If no corners are detected we impose a lower value minThFAST
+# You can lower these values if your images have low contrast			
+ORBextractor.iniThFAST: 20
+ORBextractor.minThFAST: 7
+
+#--------------------------------------------------------------------------------------------
+# Viewer Parameters
+#--------------------------------------------------------------------------------------------
+Viewer.KeyFrameSize: 0.05
+Viewer.KeyFrameLineWidth: 1
+Viewer.GraphLineWidth: 0.9
+Viewer.PointSize:2
+Viewer.CameraSize: 0.08
+Viewer.CameraLineWidth: 3
+Viewer.ViewpointX: 0
+Viewer.ViewpointY: -0.7
+Viewer.ViewpointZ: -1.8
+Viewer.ViewpointF: 500
+

GCN2/run.sh

@@ -1,8 +1,10 @@
-# GCNv2 trained with more augmentations and larger batch
-GCN_PATH=gcn2_aug.pt ./rgbd_gcn ../Vocabulary/GCNvoc.bin TUM3.yaml ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household/associations.txt
+# Run GCN, removed resolution requirment. It may affect the actual performance. Ideally, GCNv2 should be trained/finetuned in desired resolution.
+GCN_PATH=gcn2.pt ./rgbd_gcn ../Vocabulary/GCNvoc.bin TUM3.yaml ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household/associations.txt
+
+# Reproduce results in comparison with ORB as in our paper, will enable NN_ONLY and use 320x240 resolution.
 
 # GCNv2
-./rgbd_gcn ../Vocabulary/GCNvoc.bin TUM3.yaml ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household/associations.txt
+# NN_ONLY=1 GCN_PATH=gcn2.pt ./rgbd_gcn ../Vocabulary/GCNvoc.bin TUM3_small.yaml ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household/associations.txt
 
 # Vanilla ORB
-USE_ORB=1 ./rgbd_gcn ../Vocabulary/ORBvoc.bin TUM3.yaml ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household/associations.txt
+# NN_ONLY=1 USE_ORB=1 ./rgbd_gcn ../Vocabulary/ORBvoc.bin TUM3_small.yaml ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household ~/Workspace/Datasets/TUM/freiburg3/rgbd_dataset_freiburg3_long_office_household/associations.txt

src/GCNextractor.cc

@@ -74,7 +74,7 @@ const int EDGE_THRESHOLD = 19;
 
 
 void nms(cv::Mat det, cv::Mat desc, std::vector<cv::KeyPoint>& pts, cv::Mat& descriptors,
-        int border, int dist_thresh, int img_width, int img_height){
+        int border, int dist_thresh, int img_width, int img_height, float ratio_width, float ratio_height){
 
     std::vector<cv::Point2f> pts_raw;
 
@@ -135,7 +135,7 @@ void nms(cv::Mat det, cv::Mat desc, std::vector<cv::KeyPoint>& pts, cv::Mat& des
             if (grid.at<char>(v,u) == 2)
             {
                 int select_ind = (int) inds.at<unsigned short>(v-dist_thresh, u-dist_thresh);
-                pts.push_back(cv::KeyPoint(pts_raw[select_ind], 1.0f));
+                pts.push_back(cv::KeyPoint(pts_raw[select_ind].x * ratio_width, pts_raw[select_ind].y * ratio_height, 1.0f));
 
                 select_indice.push_back(select_ind);
                 valid_cnt++;
@@ -238,6 +238,11 @@ void GCNextractor::operator()( InputArray _image, InputArray _mask, vector<KeyPo
     int img_width = 320;
     int img_height = 240;
 
+    float ratio_width = float(img.cols) / float(img_width);
+    float ratio_height = float(img.rows) / float(img_height);
+
+    cv::resize(img, img, cv::Size(img_width, img_height));
+
     #if defined(TORCH_NEW_API)
         std::vector<int64_t> dims = {1, img_height, img_width, 1};
         auto img_var = torch::from_blob(img.data, dims, torch::kFloat32).to(device);
@@ -263,7 +268,7 @@ void GCNextractor::operator()( InputArray _image, InputArray _mask, vector<KeyPo
 
     std::vector<cv::KeyPoint> keypoints;
     cv::Mat descriptors;
-    nms(pts_mat, desc_mat, keypoints, descriptors, border, dist_thresh, img_width, img_height);
+    nms(pts_mat, desc_mat, keypoints, descriptors, border, dist_thresh, img_width, img_height, ratio_width, ratio_height);
 
     _keypoints.insert(_keypoints.end(), keypoints.begin(), keypoints.end());
 

src/Tracking.cc

@@ -227,8 +227,11 @@ cv::Mat Tracking::GrabImageRGBD(const cv::Mat &imRGB, const cv::Mat &imD, const
             cvtColor(mImGray,mImGray,CV_BGRA2GRAY);
     }
 
-    cv::resize(mImGray, mImGray, cv::Size(320, 240));
-    cv::resize(imDepth, imDepth, cv::Size(320, 240), 0, 0, cv::INTER_NEAREST);
+    if (getenv("NN_ONLY") != nullptr)
+    {
+        cv::resize(mImGray, mImGray, cv::Size(320, 240));
+        cv::resize(imDepth, imDepth, cv::Size(320, 240), 0, 0, cv::INTER_NEAREST);
+    }
 
     if((fabs(mDepthMapFactor-1.0f)>1e-5) || imDepth.type()!=CV_32F)
         imDepth.convertTo(imDepth,CV_32F,mDepthMapFactor);
@@ -325,20 +328,23 @@ void Tracking::Track()
                 // NN only keyframe tracking
                 bOK = TrackReferenceKeyFrame();
 
-#if 0
-                if(mVelocity.empty() || mCurrentFrame.mnId<mnLastRelocFrameId+2)
-                {
-                    bOK = TrackReferenceKeyFrame();
-                }
-                else
+                // Set NN_ONLY for neglecting other tracking heuristics of original ORB_SLAM2, used for farily compare with ORB feature
+                if (getenv("NN_ONLY") == nullptr)
                 {
-                    bOK = TrackWithMotionModel();
-                    if(!bOK)
+                    if(mVelocity.empty() || mCurrentFrame.mnId<mnLastRelocFrameId+2)
                     {
                         bOK = TrackReferenceKeyFrame();
                     }
+                    else
+                    {
+                        bOK = TrackWithMotionModel();
+                        if(!bOK)
+                        {
+                            bOK = TrackReferenceKeyFrame();
+                        }
+                    }
                 }
-#endif
+
             }
             else
             {