sync testbed GIplot code & tf session init

.gitignore

@@ -21,12 +21,11 @@ libfrvt_11_*_???.so
 11/config/09-02_02-45.pb
 11/lib/
 *.so
-*.ppm
-11/pnas
-11/GIboxPlot.png
 11/.ipynb_checkpoints/NISTplot-checkpoint.ipynb
 11/config/123/09-02_02-45.pb
 11/input/enroll.txt
 11/input/match.txt
 11/input/verif.txt
 11/src/nullImpl/.ipynb_checkpoints/GVimplfrvt11-checkpoint.cpp
+11/plot
+11/Result/

.vscode/launch.json

@@ -4,12 +4,29 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        {
+            "name": "(gdb) Attach",
+            "type": "cppdbg",
+            "request": "attach",
+            "program": "enter program name, for example ${workspaceFolder}/11/bin/validate11",
+            "processId": "${command:pickProcess}",
+            "MIMode": "gdb",
+            "setupCommands": [
+                {
+                    "description": "Enable pretty-printing for gdb",
+                    "text": "-enable-pretty-printing",
+                    "ignoreFailures": true
+                }
+            ]
+        },
         {
             "name": "Python: Current File",
             "type": "python",
             "request": "launch",
             "program": "${file}",
-            "console": "integratedTerminal"
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}",
+            "env": { "PYTHONPATH": "${workspaceRoot}"}
         }
     ]
 }

11/input/pnas/enroll.txt

@@ -1,20 +1,20 @@
-1 pnas/ppm/1.ppm MUGSHOT
-3 pnas/ppm/3.ppm MUGSHOT
-5 pnas/ppm/5.ppm MUGSHOT
-7 pnas/ppm/7.ppm MUGSHOT
-9 pnas/ppm/9.ppm MUGSHOT
-11 pnas/ppm/11.ppm MUGSHOT
-13 pnas/ppm/13.ppm MUGSHOT
-15 pnas/ppm/15.ppm MUGSHOT
-17 pnas/ppm/17.ppm MUGSHOT
-19 pnas/ppm/19.ppm MUGSHOT
-21 pnas/ppm/21.ppm MUGSHOT
-23 pnas/ppm/23.ppm MUGSHOT
-25 pnas/ppm/25.ppm MUGSHOT
-27 pnas/ppm/27.ppm MUGSHOT
-29 pnas/ppm/29.ppm MUGSHOT
-31 pnas/ppm/31.ppm MUGSHOT
-33 pnas/ppm/33.ppm MUGSHOT
-35 pnas/ppm/35.ppm MUGSHOT
-37 pnas/ppm/37.ppm MUGSHOT
-39 pnas/ppm/39.ppm MUGSHOT
+1 ../common/pnas/S001-0_1.ppm PNAS
+3 ../common/pnas/S002-0_3.ppm PNAS
+5 ../common/pnas/S003-0_5.ppm PNAS
+7 ../common/pnas/S004-0_7.ppm PNAS
+9 ../common/pnas/S005-0_9.ppm PNAS
+11 ../common/pnas/S006-0_11.ppm PNAS
+13 ../common/pnas/S007-0_13.ppm PNAS
+15 ../common/pnas/S008-0_15.ppm PNAS
+17 ../common/pnas/S009-0_17.ppm PNAS
+19 ../common/pnas/S010-0_19.ppm PNAS
+21 ../common/pnas/S011-0_21.ppm PNAS
+23 ../common/pnas/S012-0_23.ppm PNAS
+25 ../common/pnas/S013-0_25.ppm PNAS
+27 ../common/pnas/S015-0_27.ppm PNAS
+29 ../common/pnas/S017-0_29.ppm PNAS
+31 ../common/pnas/S019-0_31.ppm PNAS
+33 ../common/pnas/S021-0_33.ppm PNAS
+35 ../common/pnas/S023-0_35.ppm PNAS
+37 ../common/pnas/S025-0_37.ppm PNAS
+39 ../common/pnas/S027-0_39.ppm PNAS

11/input/pnas/verif.txt

@@ -1,20 +1,20 @@
-2 pnas/ppm/2.ppm MUGSHOT
-4 pnas/ppm/4.ppm MUGSHOT
-6 pnas/ppm/6.ppm MUGSHOT
-8 pnas/ppm/8.ppm MUGSHOT
-10 pnas/ppm/10.ppm MUGSHOT
-12 pnas/ppm/12.ppm MUGSHOT
-14 pnas/ppm/14.ppm MUGSHOT
-16 pnas/ppm/16.ppm MUGSHOT
-18 pnas/ppm/18.ppm MUGSHOT
-20 pnas/ppm/20.ppm MUGSHOT
-22 pnas/ppm/22.ppm MUGSHOT
-24 pnas/ppm/24.ppm MUGSHOT
-26 pnas/ppm/26.ppm MUGSHOT
-28 pnas/ppm/28.ppm MUGSHOT
-30 pnas/ppm/30.ppm MUGSHOT
-32 pnas/ppm/32.ppm MUGSHOT
-34 pnas/ppm/34.ppm MUGSHOT
-36 pnas/ppm/36.ppm MUGSHOT
-38 pnas/ppm/38.ppm MUGSHOT
-40 pnas/ppm/40.ppm MUGSHOT
+2 ../common/pnas/S001-0_2.ppm PNAS
+4 ../common/pnas/S002-0_4.ppm PNAS
+6 ../common/pnas/S003-0_6.ppm PNAS
+8 ../common/pnas/S004-0_8.ppm PNAS
+10 ../common/pnas/S005-0_10.ppm PNAS
+12 ../common/pnas/S006-0_12.ppm PNAS
+14 ../common/pnas/S007-0_14.ppm PNAS
+16 ../common/pnas/S008-0_16.ppm PNAS
+18 ../common/pnas/S009-0_18.ppm PNAS
+20 ../common/pnas/S010-0_20.ppm PNAS
+22 ../common/pnas/S011-0_22.ppm PNAS
+24 ../common/pnas/S012-0_24.ppm PNAS
+26 ../common/pnas/S014-0_26.ppm PNAS
+28 ../common/pnas/S016-0_28.ppm PNAS
+30 ../common/pnas/S018-0_30.ppm PNAS
+32 ../common/pnas/S020-0_32.ppm PNAS
+34 ../common/pnas/S022-0_34.ppm PNAS
+36 ../common/pnas/S024-0_36.ppm PNAS
+38 ../common/pnas/S026-0_38.ppm PNAS
+40 ../common/pnas/S028-0_40.ppm PNAS

11/plot.py

@@ -0,0 +1,57 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+
+def loadDataframe(fileDirectory):
+    # Load from a file
+    enrolldir = fileDirectory + "/enroll.log"
+    verifdir = fileDirectory + "/verif.log"
+    matchdir = fileDirectory + "/match.log"
+    dfEnroll = pd.read_csv(enrolldir, delimiter= '\s+')
+    dfVerif = pd.read_csv(verifdir, delimiter= '\s+')
+    dfMatch = pd.read_csv(matchdir, delimiter= '\s+')    
+    # Get UUID
+    imgEnrollName = ([p.strip().split('/')[3] for p in dfEnroll['image']])
+    noExtEnrollName = [n.strip(".ppm") for n in imgEnrollName]
+    enrollUUID = ([n.strip().split('-')[0] for n in noExtEnrollName]) 
+    dfEnroll['UUID'] = enrollUUID
+    imgVerifName = ([p.strip().split('/')[3] for p in dfVerif['image']])
+    noExtVerifName = [n.strip(".ppm") for n in imgVerifName]
+    verifUUID = ([n.strip().split('-')[0] for n in noExtVerifName]) 
+    dfVerif['UUID'] = verifUUID
+    # print(dfEnroll.head())
+    # print(dfVerif.head())
+    # print(dfMatch.head())
+    return dfEnroll,dfVerif,dfMatch
+
+def plotGIBoxScatter(dfGIbox):
+    #label for same id(Genuine) and different id(Imposter)
+    # ax = sns.boxplot(x="GIlabel", y="score", data=dfGIbox, showfliers = False)
+    ax = sns.swarmplot(x="GIlabel", y="score", data=dfGIbox, hue="returnCode")
+    minVal = dfGIbox['score'].min()
+    maxVal = dfGIbox['score'].max()
+    tickLen = float(maxVal - minVal)/20.0
+    plt.yticks(np.arange(minVal, maxVal, tickLen))
+    plt.grid()
+    plt.savefig('GIboxPlot.png')
+    plt.show()
+    return plt
+
+#load image list
+dfEnroll, dfVerif, dfMatch = loadDataframe('validation')
+
+#match pairs similarity score
+dfGIbox = pd.DataFrame(columns=['GIlabel','score','returnCode'])
+for i in range(len(dfMatch)):
+    dfGIbox.at[i,'score'] = dfMatch.at[i,'simScore']
+    if dfEnroll.at[i, 'returnCode'] != 0 or dfVerif.at[i, 'returnCode'] != 0:
+        dfGIbox.at[i, 'returnCode'] = 'FaceDetectionError'
+    else:
+        dfGIbox.at[i, 'returnCode'] = 'MatchSuccess'
+    if dfEnroll.at[i, 'UUID'] == dfVerif.at[i, 'UUID']:
+        dfGIbox.at[i, 'GIlabel'] = 'Genuine'
+    else:
+        dfGIbox.at[i, 'GIlabel'] = 'Imposter'
+# print(dfGIbox)
+plotGIBoxScatter(dfGIbox)

11/requirements.txt

@@ -0,0 +1,3 @@
+numpy==1.15.4
+matplotlib
+seaborn

11/src/nullImpl/GVimplfrvt11.cpp

@@ -30,6 +30,7 @@ NullImplFRVT11::~NullImplFRVT11() {
     // SCOPE_EXIT{ tf_utils::DeleteGraph(graphFD); }; // Auto-delete on scope exit.
     // face_input_detector = dlib::get_frontal_face_detector();
     //For FR
+    SCOPE_EXIT{ tf_utils::DeleteSession(session); }; // Auto-delete on scope exit.eleteGraph(graph); }; // Auto-delete on scope exit.
     SCOPE_EXIT{ tf_utils::DeleteGraph(graph); }; // Auto-delete on scope exit.
 }
 
@@ -75,6 +76,7 @@ NullImplFRVT11::initialize(const std::string &configDir)
         }
         mean_values[0]  = mean_values[1]  = mean_values[2]  = 255.0*0.5;
 	    scale_values[0] = scale_values[1] = scale_values[2] = 255.0*0.5;
+        session = NULL;
         // -----------------------------------------------------------------------------------------------------
     } catch (const std::exception & ex) {
         std::cerr << ex.what() << std::endl;
@@ -174,23 +176,23 @@ NullImplFRVT11::createTemplate(
                     // << face_det[0].right() << ","<< face_det[0].top() << "]"<< std::endl;
 
                 //show the score of the face. Its range is [0-100]
-                char sScore[256];
-                snprintf(sScore, 256, "%d", confidence);
-                cv::putText(result_image, sScore, cv::Point(x, y-3), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
-                //draw face rectangle
-                rectangle(result_image, cv::Rect(x, y, w, h), cv::Scalar(0, 255, 0), 2);
-                //draw five face landmarks in different colors
-                cv::circle(result_image, cv::Point(int(p[5]*Ratio), int(p[5 + 1]*Ratio)), 1, cv::Scalar(255, 0, 0), 2);
-                cv::circle(result_image, cv::Point(int(p[5 + 2]*Ratio), int(p[5 + 3]*Ratio)), 1, cv::Scalar(0, 0, 255), 2);
-                cv::circle(result_image, cv::Point(int(p[5 + 4]*Ratio), int(p[5 + 5]*Ratio)), 1, cv::Scalar(0, 255, 0), 2);
-                cv::circle(result_image, cv::Point(int(p[5 + 6]*Ratio), int(p[5 + 7]*Ratio)), 1, cv::Scalar(255, 0, 255), 2);
-                cv::circle(result_image, cv::Point(int(p[5 + 8]*Ratio), int(p[5 + 9]*Ratio)), 1, cv::Scalar(0, 255, 255), 2);
+                // char sScore[256];
+                // snprintf(sScore, 256, "%d", confidence);
+                // cv::putText(result_image, sScore, cv::Point(x, y-3), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
+                // //draw face rectangle
+                // rectangle(result_image, cv::Rect(x, y, w, h), cv::Scalar(0, 255, 0), 2);
+                // //draw five face landmarks in different colors
+                // cv::circle(result_image, cv::Point(int(p[5]*Ratio), int(p[5 + 1]*Ratio)), 1, cv::Scalar(255, 0, 0), 2);
+                // cv::circle(result_image, cv::Point(int(p[5 + 2]*Ratio), int(p[5 + 3]*Ratio)), 1, cv::Scalar(0, 0, 255), 2);
+                // cv::circle(result_image, cv::Point(int(p[5 + 4]*Ratio), int(p[5 + 5]*Ratio)), 1, cv::Scalar(0, 255, 0), 2);
+                // cv::circle(result_image, cv::Point(int(p[5 + 6]*Ratio), int(p[5 + 7]*Ratio)), 1, cv::Scalar(255, 0, 255), 2);
+                // cv::circle(result_image, cv::Point(int(p[5 + 8]*Ratio), int(p[5 + 9]*Ratio)), 1, cv::Scalar(0, 255, 255), 2);
 
-                cv::putText(result_image, "0", cv::Point(int(p[5]*Ratio), int(p[5 + 1]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
-                cv::putText(result_image, "1", cv::Point(int(p[5 + 2]*Ratio), int(p[5 + 3]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
-                cv::putText(result_image, "2", cv::Point(int(p[5 + 4]*Ratio), int(p[5 + 5]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
-                cv::putText(result_image, "3", cv::Point(int(p[5 + 6]*Ratio), int(p[5 + 7]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
-                cv::putText(result_image, "4", cv::Point(int(p[5 + 8]*Ratio), int(p[5 + 9]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
+                // cv::putText(result_image, "0", cv::Point(int(p[5]*Ratio), int(p[5 + 1]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
+                // cv::putText(result_image, "1", cv::Point(int(p[5 + 2]*Ratio), int(p[5 + 3]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
+                // cv::putText(result_image, "2", cv::Point(int(p[5 + 4]*Ratio), int(p[5 + 5]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
+                // cv::putText(result_image, "3", cv::Point(int(p[5 + 6]*Ratio), int(p[5 + 7]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
+                // cv::putText(result_image, "4", cv::Point(int(p[5 + 8]*Ratio), int(p[5 + 9]*Ratio)), cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(0, 255, 0), 1);
 
 
                 // int eyeWidth = int(abs((p[5]*2 - p[5 + 2]))*0.25)
@@ -325,16 +327,16 @@ NullImplFRVT11::createTemplate(
                 dlib::assign_image(enroll_chipBGR, enroll_chip);
 
                 //for Bossun Debug save chip
-                cv::Mat enrollChipMat = dlib::toMat(enroll_chip);
-                debugImgMtx.lock();
-                std::time_t t = std::time(0);   // get time now
-                now = std::localtime(&t);
-                std::srand((unsigned) time(&t));
-                int randNum = rand() % 10000;
-                char chipFileName[512];
-				sprintf(chipFileName,"features/chip(%d)_(%d).jpg",faceFeatureCount,randNum);
-                dlib::save_jpeg(enroll_chip,chipFileName,100);
-                debugImgMtx.unlock();
+                // cv::Mat enrollChipMat = dlib::toMat(enroll_chip);
+                // debugImgMtx.lock();
+                // std::time_t t = std::time(0);   // get time now
+                // now = std::localtime(&t);
+                // std::srand((unsigned) time(&t));
+                // int randNum = rand() % 10000;
+                // char chipFileName[512];
+				// sprintf(chipFileName,"features/chip(%d)_(%d).jpg",faceFeatureCount,randNum);
+                // dlib::save_jpeg(enroll_chip,chipFileName,100);
+                // debugImgMtx.unlock();
                 //for Bossun Debug save chip
 
                 // // cv::imwrite(chipFileName, enrollChipMat);
@@ -473,14 +475,15 @@ NullImplFRVT11::createTemplate(
                 TF_SessionOptions* options = TF_NewSessionOptions();
                 std::array<std::uint8_t, 13> config = {{ 0x0a ,0x07, 0x0a, 0x03, 0x43, 0x50, 0x55, 0x10, 0x01, 0x10, 0x01, 0x28, 0x01}};
                 TF_SetConfig(options, config.data(), config.size(), status);
-                TF_Session* session = tf_utils::CreateSession( graph, options, status );
+                if(session == NULL){
+                    session = tf_utils::CreateSession( graph, options, status );
+                }
                 // run Session
                 clock_t beginFR = clock();
                 const TF_Code code = tf_utils::RunSession( session, input_ops, input_tensors, out_ops, output_tensors );
                 clock_t endFR = clock();
                 // double time_spentFR = (double)(endFR - beginFR) / CLOCKS_PER_SEC;
                 // std::cout << "[INFO] FR TF execute time: "<<time_spentFR<< " sec spent" << std::endl;
-                SCOPE_EXIT{ tf_utils::DeleteSession(session); }; // Auto-delete on scope exit.
                 // get the data:
                 const std::vector<std::vector<float>> dataOutputResults = tf_utils::GetTensorsData<float>( output_tensors );
 

11/src/nullImpl/GVimplfrvt11.h

@@ -162,6 +162,7 @@ namespace FRVT_11 {
 
     //====================For FR====================//
     TF_Graph *graph;
+    TF_Session* session;
     // InferenceEngine::Core engine_ptr;
 	// InferenceEngine::InferRequest infer_request;
     // InferenceEngine::CNNNetwork network;

README.md

@@ -1,26 +1,28 @@
-Add python code to validate our api implement with FAR & FRR:
-[validate.py](11/validate.py)
+Python code to test NIST frvt accuracy of MUGSHOT & pnas datasets
 
 ```
 cd 11
-run ./compileAndRunFrvt.sh first
+./compileAndRunFrvt.sh
+pip install -r requirements.txt
+python plot.py
 ```
 
-[SUCCESS] NIST frvt validation for confidence:  0.73 
+* NIST comes with the MUGSHOT dataset for testing:
+  * 653 pairs (1306 img ppm files)
+* pnas is also used for quick verification:
+  * 20 pairs (12 Genuine(G) pairs & 8 Imposter(I) pairs)
+* Additional tensorflow FR model & lib/*.so need to be downloaded seperately:
+  * 09-02_02-45.pb
 
-All count:  653 
+## G-I Similarity box scatter chart
 
-Known:  325 
+* MUGSHOT:
 
-Unknown:  264 
+![Alt text](11/GIboxPlot.png?raw=true "Title")
 
-knownToUnknown:  64 
+* pnas:
 
-unknownToKnown 0 
-
-FAR:  0.0 %
-
-FRR:  9.80091883614089 %
+![Alt text](11/GIboxPlotPNAS.png?raw=true "Title")
 
 
 # Face Recognition Vendor Test (FRVT) Validation Packages