[Bug Fix] release task scripts (PaddlePaddle#411)

* Update py_run.bat * Update cpp_run.bat * Update compare_with_gt.py Increase score_diff and boxes_diff_ratio threshold * Update cpp_run.bat * Update release task scripts according to diffrent platforms * Delete CMAKE_CXX_COMPILER in cpp_run.bat
xinzaifeixiang1992 · Oct 22, 2022 · f4c32a6 · f4c32a6
1 parent 43d8611
commit f4c32a6
Show file tree

Hide file tree

Showing 5 changed files with 57 additions and 23 deletions.
diff --git a/tests/release_task/compare_with_gt.py b/tests/release_task/compare_with_gt.py
@@ -1,6 +1,39 @@
 import numpy as np
 import re
 
+diff_score_threshold = {
+    "linux-x64": {
+        "label_diff": 0,
+        "score_diff": 1e-4,
+        "boxes_diff_ratio": 1e-4,
+        "boxes_diff": 1e-3
+    },
+    "linux-aarch64": {
+        "label_diff": 0,
+        "score_diff": 1e-4,
+        "boxes_diff_ratio": 1e-4,
+        "boxes_diff": 1e-3
+    },
+    "osx-x86_64": {
+        "label_diff": 0,
+        "score_diff": 1e-4,
+        "boxes_diff_ratio": 2e-4,
+        "boxes_diff": 1e-3
+    },
+    "osx-arm64": {
+        "label_diff": 0,
+        "score_diff": 1e-4,
+        "boxes_diff_ratio": 2e-4,
+        "boxes_diff": 1e-3
+    },
+    "win-x64": {
+        "label_diff": 0,
+        "score_diff": 5e-4,
+        "boxes_diff_ratio": 1e-3,
+        "boxes_diff": 1e-3
+    }
+}
+
 
 def parse_arguments():
     import argparse
@@ -59,6 +92,7 @@ def save_numpy_result(file_path, error_msg):
 
 
 def check_result(gt_result, infer_result, args):
+    platform = args.platform
     if len(gt_result) != len(infer_result):
         infer_result = infer_result[-len(gt_result):]
     diff = np.abs(gt_result - infer_result)
@@ -68,17 +102,19 @@ def check_result(gt_result, infer_result, args):
     boxes_diff_ratio = boxes_diff / (infer_result[:, :-2] + 1e-6)
     is_diff = False
     backend = args.result_path.split(".")[0]
-    if (label_diff > 0).any():
+    if (label_diff > diff_score_threshold[platform]["label_diff"]).any():
         print(args.platform, args.device, "label diff ", label_diff)
         is_diff = True
         label_diff_bool_file = args.platform + "_" + backend + "_" + "label_diff_bool.txt"
         save_numpy_result(label_diff_bool_file, label_diff > 0)
-    if (score_diff > 2e-4).any():
+    if (score_diff > diff_score_threshold[platform]["score_diff"]).any():
         print(args.platform, args.device, "score diff ", score_diff)
         is_diff = True
         score_diff_bool_file = args.platform + "_" + backend + "_" + "score_diff_bool.txt"
         save_numpy_result(score_diff_bool_file, score_diff > 1e-4)
-    if (boxes_diff_ratio > 1e-4).any() and (boxes_diff > 1e-3).any():
+    if (boxes_diff_ratio > diff_score_threshold[platform]["boxes_diff_ratio"]
+        ).any() and (
+            boxes_diff > diff_score_threshold[platform]["boxes_diff"]).any():
         print(args.platform, args.device, "boxes diff ", boxes_diff_ratio)
         is_diff = True
         boxes_diff_bool_file = args.platform + "_" + backend + "_" + "boxes_diff_bool.txt"

diff --git a/tests/release_task/cpp_run.bat b/tests/release_task/cpp_run.bat
@@ -17,7 +17,7 @@ if "%DEVICE%" == "gpu" (
     set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%DEVICE%-%VERSION%
     set RUN_CASES=ort paddle trt
 ) else (
-    set CPP_FASTDEPLOY_PACKAGE=fastdeploy-python
+    set CPP_FASTDEPLOY_PACKAGE=fastdeploy-%PLATFORM%-%VERSION%
     set RUN_CASES=ort paddle openvino
 )
 
@@ -36,7 +36,7 @@ set COMPARE_SHELL=%CURRENT_DIR%\compare_with_gt.py
 python -c "from download import *; download_and_decompress('https://fastdeploy.bj.bcebos.com/dev/cpp/%CPP_FASTDEPLOY_PACKAGE%.zip', './')"
 
 mkdir build && cd build
-cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2"  -DCMAKE_CXX_COMPILER=%CMAKE_CXX_COMPILER%
+cmake .. -G "Visual Studio 16 2019" -A x64 -DFASTDEPLOY_INSTALL_DIR=%cd%\..\%CPP_FASTDEPLOY_PACKAGE% -DCUDA_DIRECTORY="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.2"
 
 msbuild infer_demo.sln /m:4 /p:Configuration=Release /p:Platform=x64
 
@@ -47,7 +47,6 @@ echo "FASTDEPLOY_HOME" %FASTDEPLOY_HOME%
 copy /Y %FASTDEPLOY_HOME%\third_libs\install\onnxruntime\lib\onnxruntime* Release\
 set PATH=%FASTDEPLOY_HOME%\lib;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\onnxruntime\lib;%PATH%
-set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv-win-x64-3.4.16\build\x64\vc15\bin;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\paddle\lib;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mkldnn\lib;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\paddle_inference\third_party\install\mklml\lib;%PATH%
@@ -56,9 +55,9 @@ set PATH=%FASTDEPLOY_HOME%\third_libs\install\tensorrt\lib;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\lib;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\faster_tokenizer\third_party\lib;%PATH%
 set PATH=%FASTDEPLOY_HOME%\third_libs\install\yaml-cpp\lib;%PATH%
-set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\bin;%PATH%
-set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\3rdparty\tbb\bin;%PATH%
-
+set PATH=%FASTDEPLOY_HOME%\third_libs\install\opencv\build\x64\vc15\bin;%PATH%
+set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\bin;%PATH%
+set PATH=%FASTDEPLOY_HOME%\third_libs\install\openvino\runtime\3rdparty\tbb\bin;%PATH%
 echo "set path done"
 cd %cd%\Release
 for %%b in (%RUN_CASES%) do (
@@ -71,11 +70,9 @@ for %%b in (%RUN_CASES%) do (
         if %%b == trt (
             infer_ppyoloe_demo.exe --model_dir=%MODEL_PATH% --image_file=%IMAGE_PATH% --device=gpu --backend=%%b >> cpp_%%b_trt_result.txt
             python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path cpp_%%b_trt_result.txt --platform %PLATFORM% --device trt --conf_threshold 0.5
-        ) else if %%b == ort (
+        ) else (
             infer_ppyoloe_demo.exe --model_dir=%MODEL_PATH% --image_file=%IMAGE_PATH% --device=gpu --backend=%%b >> cpp_%%b_gpu_result.txt
             python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path cpp_%%b_gpu_result.txt --platform %PLATFORM% --device gpu --conf_threshold 0.5
-        ) else if %%b == paddle (
-            echo "Temporarily skip paddle gpu case in windows for Inaccurate inference precision" 
         )
     ) 
 )

diff --git a/tests/release_task/cpp_run.sh b/tests/release_task/cpp_run.sh
@@ -12,7 +12,8 @@ fi
 echo $CPP_FASTDEPLOY_PACKAGE
 LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt')
 LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino')
-LINUX_AARCH_CPU_CASE=('ort' 'openvino')
+#LINUX_AARCH_CPU_CASE=('ort' 'openvino')
+LINUX_AARCH_CPU_CASE=('ort')
 MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino')
 MACOS_ARM64_CPU_CASE=('default')
 wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
@@ -25,6 +26,7 @@ GROUND_TRUTH_PATH=$CURRENT_DIR/release_task_groud_truth_result.txt
 COMPARE_SHELL=$CURRENT_DIR/compare_with_gt.py
 
 RUN_CASE=()
+CONF_THRESHOLD=0
 if [ "$DEVICE" = "gpu" ] && [ "$PLATFORM" = "linux-x64" ];then
 	RUN_CASE=(${LINUX_X64_GPU_CASE[*]})
 elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "linux-x64" ];then
@@ -35,6 +37,7 @@ elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-x86_64" ];then
 	RUN_CASE=(${MACOS_INTEL_CPU_CASE[*]})
 elif [ "$DEVICE" = "cpu" ] && [ "$PLATFORM" = "osx-arm64" ];then
 	RUN_CASE=(${MACOS_ARM64_CPU_CASE[*]})
+	CONF_THRESHOLD=0.5
 fi
 
 case_number=${#RUN_CASE[@]}
@@ -52,16 +55,16 @@ do
        echo "Cpp Backend:" $backend
        if [ "$backend" != "trt" ];then
                ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=cpu --backend=$backend >> cpp_cpu_result.txt
-               python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu
+               python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_cpu_result.txt --platform $PLATFORM --device cpu --conf_threshold $CONF_THRESHOLD
        fi
        if [ "$DEVICE" = "gpu" ];then
 
 	       if [ "$backend" = "trt" ];then
                        ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_trt_result.txt
-                       python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt
+                       python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_trt_result.txt --platform $PLATFORM --device trt --conf_threshold $CONF_THRESHOLD
 	       else
                        ./infer_ppyoloe_demo --model_dir=$MODEL_PATH --image_file=$IMAGE_PATH --device=gpu --backend=$backend >> cpp_gpu_result.txt
-                       python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu
+                       python $COMPARE_SHELL --gt_path $GROUND_TRUTH_PATH --result_path cpp_gpu_result.txt --platform $PLATFORM --device gpu --conf_threshold $CONF_THRESHOLD
                fi
        fi
 done
@@ -78,4 +81,3 @@ else
        cat $res_file
        exit -1
 fi
-
diff --git a/tests/release_task/py_run.bat b/tests/release_task/py_run.bat
@@ -46,12 +46,10 @@ for %%b in (%RUN_CASES%) do (
         if %%b == trt (
             python infer_ppyoloe.py --model_dir=%MODEL_PATH% --image=%IMAGE_PATH% --device=gpu --backend=%%b >> py_%%b_trt_result.txt 
             python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path py_%%b_trt_result.txt --platform %PLATFORM% --device trt --conf_threshold 0.5
-        ) else if  %%b == ort (
+        ) else (
             python infer_ppyoloe.py --model_dir=%MODEL_PATH% --image=%IMAGE_PATH% --device=gpu --backend=%%b >> py_%%b_gpu_result.txt 
             python %COMPARE_SHELL% --gt_path %GROUND_TRUTH_PATH% --result_path py_%%b_gpu_result.txt --platform %PLATFORM% --device gpu  --conf_threshold 0.5
-        ) else if  %%b == paddle (
-	    echo "Temporarily skip paddle gpu case in windows for Inaccurate inference precision"
-	)
+        )
     ) 
 )
 

diff --git a/tests/release_task/py_run.sh b/tests/release_task/py_run.sh
@@ -12,7 +12,8 @@ fi
 echo $PY_FASTDEPLOY_PACKAGE
 LINUX_X64_GPU_CASE=('ort' 'paddle' 'trt')
 LINUX_X64_CPU_CASE=('ort' 'paddle' 'openvino')
-LINUX_AARCH_CPU_CASE=('ort' 'openvino')
+#LINUX_AARCH_CPU_CASE=('ort' 'openvino')
+LINUX_AARCH_CPU_CASE=('ort')
 MACOS_INTEL_CPU_CASE=('ort' 'paddle' 'openvino')
 MACOS_ARM64_CPU_CASE=('default')
 wget -q https://bj.bcebos.com/paddlehub/fastdeploy/ppyoloe_crn_l_300e_coco.tgz
@@ -60,7 +61,7 @@ do
        fi
 done
 
-ret=$? 
+ret=$?
 
 res_file="result.txt"
 if [ ! -f $res_file ];then