[Android] Support segmentation and facedet in Android (PaddlePaddle#567)

* [FlyCV] Add global SetProcLibCpuNumThreads method

* [Android] Support segmentation and facedet in Android

* [Android] add JNI instance check to j_runtime_option_obj

* [Model] fixed ppseg flycv resize error

* [FlyCV] fix FlyCV resize flags

* [cmake] remove un-need lite compile option

* [Android] add PaddleSegModel JNI and fix some bugs

* [Android] bind PaddleSegModel via JNI

* [Android] bind VisSegmentation via JNI

* [Android] bind YOLOv5Face and SCRFD via JNI

* [Android] fix NewJavaFaceDetectionResultFromCxx error

CMakeLists.txt

@@ -63,14 +63,11 @@ option(ENABLE_LITE_BACKEND "Whether to enable paddle lite backend." OFF)
 option(ENABLE_VISION "Whether to enable vision models usage." OFF)
 option(ENABLE_TEXT "Whether to enable text models usage." OFF)
 option(ENABLE_FLYCV "Whether to enable flycv to boost image preprocess." OFF)
+option(ENABLE_TIMVX "Whether to compile for TIMVX deploy." OFF)
 option(WITH_TESTING "Whether to compile with unittest." OFF)
 ############################# Options for Android cross compiling #########################
 option(WITH_OPENCV_STATIC "Use OpenCV static lib for Android." OFF)
 option(WITH_LITE_STATIC "Use Paddle Lite static lib for Android." OFF)
-option(WITH_LITE_FULL_API "Use Paddle Lite full API lib for Android." ON)
-option(WITH_LITE_FP16 "Use Paddle Lite lib with fp16 enabled for Android." OFF)
-
-option(ENABLE_TIMVX "Whether to compile for TIMVX deploy." OFF)
 
 # Please don't open this flag now, some bugs exists.
 # Only support Linux Now

cmake/paddlelite.cmake

@@ -40,9 +40,6 @@ if(ANDROID)
   if(WITH_LITE_STATIC)
       message(FATAL_ERROR "Doesn't support WTIH_LITE_STATIC=ON for Paddle Lite now.")
   endif()  
-  if(NOT WITH_LITE_FULL_API)
-      message(FATAL_ERROR "Doesn't support WITH_LITE_FULL_API=OFF for Paddle Lite now.")
-  endif()
   # check ABI, toolchain
   if((NOT ANDROID_ABI MATCHES "armeabi-v7a") AND (NOT ANDROID_ABI MATCHES "arm64-v8a"))
     message(FATAL_ERROR "FastDeploy with Paddle Lite only support armeabi-v7a, arm64-v8a now.")
@@ -56,13 +53,9 @@ if(WIN32 OR APPLE OR IOS)
   message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle Lite now.")
 elseif(ANDROID)
   set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-latest.tgz")
-  if(WITH_LITE_FP16) 
-    if(ANDROID_ABI MATCHES "arm64-v8a") 
-      set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-fp16-latest.tgz")
-    else()
-      message(FATAL_ERROR "Doesn't support fp16 for ${ANDROID_ABI} now !")
-    endif()
-  endif()
+  if(ANDROID_ABI MATCHES "arm64-v8a") 
+    set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-android-${ANDROID_ABI}-fp16-latest.tgz")
+  endif()  
 else() # Linux
   if(CMAKE_HOST_SYSTEM_PROCESSOR MATCHES "aarch64")
     set(PADDLELITE_URL "${PADDLELITE_URL_PREFIX}/lite-linux-arm64-20220920.tgz")
@@ -76,21 +69,11 @@ endif()
 if(WIN32 OR APPLE OR IOS)
   message(FATAL_ERROR "Doesn't support windows/mac/ios platform with backend Paddle Lite now.")
 elseif(ANDROID AND WITH_LITE_STATIC)
-  if(WITH_LITE_FULL_API)
-    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled..a")
-    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a")
-  else()
-    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a")
-    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled.a")
-  endif()  
+  set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_full_bundled..a")
+  set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_api_light_bundled.a")
 else()
-  if(WITH_LITE_FULL_API)
-    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so")
-    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so")
-  else()
-    set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so")
-    set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so")
-  endif()  
+  set(PADDLELITE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_full_api_shared.so")
+  set(PADDLELITE_REMOVE_LIB "${PADDLELITE_LIB_DIR}/libpaddle_light_api_shared.so")
 endif()
 
 include_directories(${PADDLELITE_INC_DIR})

fastdeploy/vision/common/processors/limit_by_stride.cc

@@ -58,8 +58,10 @@ bool LimitByStride::ImplByFlyCV(Mat* mat) {
       interp_method = fcv::InterpolationType::INTER_LINEAR;
     } else if (interp_ == 2) {
       interp_method = fcv::InterpolationType::INTER_CUBIC;
+    } else if (interp_ == 3) {
+      interp_method = fcv::InterpolationType::INTER_AREA; 
     } else {
-      FDERROR << "LimitByStride: Only support interp_ be 0/1/2 with FlyCV, but "
+      FDERROR << "LimitByStride: Only support interp_ be 0/1/2/3 with FlyCV, but "
                  "now it's "
               << interp_ << "." << std::endl;
       return false;

fastdeploy/vision/common/processors/limit_short.cc

@@ -64,8 +64,10 @@ bool LimitShort::ImplByFlyCV(Mat* mat) {
       interp_method = fcv::InterpolationType::INTER_LINEAR;
     } else if (interp_ == 2) {
       interp_method = fcv::InterpolationType::INTER_CUBIC;
+    } else if (interp_ == 3) {
+      interp_method = fcv::InterpolationType::INTER_AREA; 
     } else {
-      FDERROR << "LimitByShort: Only support interp_ be 0/1/2 with FlyCV, but "
+      FDERROR << "LimitByShort: Only support interp_ be 0/1/2/3 with FlyCV, but "
                  "now it's "
               << interp_ << "." << std::endl;
       return false;

fastdeploy/vision/common/processors/resize.cc

@@ -78,8 +78,10 @@ bool Resize::ImplByFlyCV(Mat* mat) {
     interp_method = fcv::InterpolationType::INTER_LINEAR;
   } else if (interp_ == 2) {
     interp_method = fcv::InterpolationType::INTER_CUBIC;
+  } else if (interp_ == 3) {
+    interp_method = fcv::InterpolationType::INTER_AREA;  
   } else {
-    FDERROR << "Resize: Only support interp_ be 0/1/2 with FlyCV, but "
+    FDERROR << "Resize: Only support interp_ be 0/1/2/3 with FlyCV, but "
                "now it's "
             << interp_ << "." << std::endl;
     return false;

fastdeploy/vision/common/processors/resize_by_short.cc

@@ -50,8 +50,10 @@ bool ResizeByShort::ImplByFlyCV(Mat* mat) {
     interp_method = fcv::InterpolationType::INTER_LINEAR;
   } else if (interp_ == 2) {
     interp_method = fcv::InterpolationType::INTER_CUBIC;
+  } else if (interp_ == 3) {
+    interp_method = fcv::InterpolationType::INTER_AREA; 
   } else {
-    FDERROR << "LimitByShort: Only support interp_ be 0/1/2 with FlyCV, but "
+    FDERROR << "LimitByShort: Only support interp_ be 0/1/2/3 with FlyCV, but "
                "now it's "
             << interp_ << "." << std::endl;
     return false;

fastdeploy/vision/detection/contrib/scaledyolov4.cc

@@ -114,7 +114,7 @@ bool ScaledYOLOv4::Preprocess(
   // process after image load
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolor.cc

@@ -112,7 +112,7 @@ bool YOLOR::Preprocess(Mat* mat, FDTensor* output,
   // process after image load
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolov5.cc

@@ -145,7 +145,7 @@ bool YOLOv5::Preprocess(Mat* mat, FDTensor* output,
   // process after image load
   double ratio = (size[0] * 1.0) / std::max(static_cast<float>(mat->Height()),
                                             static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolov5lite.cc

@@ -164,7 +164,7 @@ bool YOLOv5Lite::Preprocess(
   // process after image load
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolov6.cc

@@ -136,7 +136,7 @@ bool YOLOv6::Preprocess(Mat* mat, FDTensor* output,
   // process after image load
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolov7.cc

@@ -134,7 +134,7 @@ bool YOLOv7::Preprocess(Mat* mat, FDTensor* output,
   // process after image load
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolov7end2end_ort.cc

@@ -115,7 +115,7 @@ bool YOLOv7End2EndORT::Preprocess(
     std::map<std::string, std::array<float, 2>>* im_info) {
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/detection/contrib/yolov7end2end_trt.cc

@@ -146,7 +146,7 @@ bool YOLOv7End2EndTRT::Preprocess(
     std::map<std::string, std::array<float, 2>>* im_info) {
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;

fastdeploy/vision/facedet/contrib/scrfd.cc

@@ -66,7 +66,7 @@ SCRFD::SCRFD(const std::string& model_file, const std::string& params_file,
     valid_cpu_backends = {Backend::ORT};
     valid_gpu_backends = {Backend::ORT, Backend::TRT};  
   } else {
-    valid_cpu_backends = {Backend::PDINFER, Backend::ORT};
+    valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE};
     valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
     valid_rknpu_backends = {Backend::RKNPU2};
   }
@@ -119,7 +119,11 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output,
                        std::map<std::string, std::array<float, 2>>* im_info) {
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {
+ #ifndef __ANDROID__  
+  // Because of the low CPU performance on the Android device, 
+  // we decided to hide this extra resize. It won't make much 
+  // difference to the final result.
+  if (std::fabs(ratio - 1.0f) > 1e-06) {
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;
@@ -128,6 +132,7 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output,
     int resize_w = int(mat->Width() * ratio);
     Resize::Run(mat, resize_w, resize_h, -1, -1, interp);
   }
+#endif  
   // scrfd's preprocess steps
   // 1. letterbox
   // 2. BGR->RGB
@@ -153,7 +158,7 @@ bool SCRFD::Preprocess(Mat* mat, FDTensor* output,
   (*im_info)["output_shape"] = {static_cast<float>(mat->Height()),
                                 static_cast<float>(mat->Width())};
   mat->ShareWithTensor(output);
-  output->shape.insert(output->shape.begin(), 1);  // reshape to n, h, w, c
+  output->shape.insert(output->shape.begin(), 1);  // reshape to n, c, h, w
   return true;
 }
 
@@ -226,7 +231,13 @@ bool SCRFD::Postprocess(
     pad_w = static_cast<float>(static_cast<int>(pad_w) % stride);
   }
   // must be setup landmarks_per_face before reserve
-  result->landmarks_per_face = landmarks_per_face;
+  if (use_kps) {
+    result->landmarks_per_face = landmarks_per_face;
+  } else {
+    // force landmarks_per_face = 0, if use_kps has been set as 'false'.
+    result->landmarks_per_face = 0; 
+  }
+
   result->Reserve(total_num_boxes);
   unsigned int count = 0;
   // loop each stride
@@ -310,11 +321,13 @@ bool SCRFD::Postprocess(
     result->boxes[i][3] = std::min(result->boxes[i][3], ipt_h - 1.0f);
   }
   // scale and clip landmarks
-  for (size_t i = 0; i < result->landmarks.size(); ++i) {
-    result->landmarks[i][0] = std::max(result->landmarks[i][0], 0.0f);
-    result->landmarks[i][1] = std::max(result->landmarks[i][1], 0.0f);
-    result->landmarks[i][0] = std::min(result->landmarks[i][0], ipt_w - 1.0f);
-    result->landmarks[i][1] = std::min(result->landmarks[i][1], ipt_h - 1.0f);
+  if (use_kps) {
+      for (size_t i = 0; i < result->landmarks.size(); ++i) {
+        result->landmarks[i][0] = std::max(result->landmarks[i][0], 0.0f);
+        result->landmarks[i][1] = std::max(result->landmarks[i][1], 0.0f);
+        result->landmarks[i][0] = std::min(result->landmarks[i][0], ipt_w - 1.0f);
+        result->landmarks[i][1] = std::min(result->landmarks[i][1], ipt_h - 1.0f);
+    }
   }
   return true;
 }

fastdeploy/vision/facedet/contrib/yolov5face.cc

@@ -67,7 +67,7 @@ YOLOv5Face::YOLOv5Face(const std::string& model_file,
     valid_cpu_backends = {Backend::ORT}; 
     valid_gpu_backends = {Backend::ORT, Backend::TRT}; 
   } else {
-    valid_cpu_backends = {Backend::PDINFER, Backend::ORT};
+    valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE};
     valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT};
   }
   runtime_option = custom_option;
@@ -115,7 +115,11 @@ bool YOLOv5Face::Preprocess(
   // process after image load
   float ratio = std::min(size[1] * 1.0f / static_cast<float>(mat->Height()),
                          size[0] * 1.0f / static_cast<float>(mat->Width()));
-  if (ratio != 1.0) {  // always true
+#ifndef __ANDROID__     
+  // Because of the low CPU performance on the Android device, 
+  // we decided to hide this extra resize. It won't make much 
+  // difference to the final result.
+  if (std::fabs(ratio - 1.0f) > 1e-06) {  
     int interp = cv::INTER_AREA;
     if (ratio > 1.0) {
       interp = cv::INTER_LINEAR;
@@ -124,6 +128,7 @@ bool YOLOv5Face::Preprocess(
     int resize_w = int(round(static_cast<float>(mat->Width()) * ratio));
     Resize::Run(mat, resize_w, resize_h, -1, -1, interp);
   }
+#endif  
   // yolov5face's preprocess steps
   // 1. letterbox
   // 2. BGR->RGB

fastdeploy/vision/segmentation/ppseg/model.cc

@@ -263,8 +263,8 @@ bool PaddleSegModel::Postprocess(
           infer_result->shape, FDDataType::FP32,
           static_cast<void*>(fp32_result_buffer->data()));
     }
-    mat = new Mat(Mat::Create(*infer_result));
-    Resize::Run(mat, ipt_w, ipt_h, -1.0f, -1.0f, 1);
+    mat = new Mat(Mat::Create(*infer_result, ProcLib::OPENCV));
+    Resize::Run(mat, ipt_w, ipt_h, -1.0f, -1.0f, 1, false, ProcLib::OPENCV);
     mat->ShareWithTensor(&new_infer_result);
     result->shape = new_infer_result.shape;
   } else {

java/android/app/src/main/AndroidManifest.xml

@@ -11,7 +11,7 @@
     <application
         android:allowBackup="true"
         android:icon="@mipmap/ic_launcher"
-        android:label="@string/ocr_app_name"
+        android:label="@string/detection_app_name"
         android:roundIcon="@mipmap/ic_launcher_round"
         android:supportsRtl="true"
         android:theme="@style/AppTheme">

java/android/app/src/main/java/com/baidu/paddle/fastdeploy/app/examples/ocr/OcrMainActivity.java

@@ -205,15 +205,13 @@ public void initSettings() {
     }
 
     public void checkAndUpdateSettings() {
-        if (OcrSettingsActivity.checkAndUpdateSettings(this)) {
-            String realModelDir = getCacheDir() + "/" + OcrSettingsActivity.modelDir;
-            // String detModelName = "ch_PP-OCRv2_det_infer";
-            String detModelName = "ch_PP-OCRv3_det_infer";
+        if (SettingsActivity.checkAndUpdateSettings(this)) {
+            String realModelDir = getCacheDir() + "/" + SettingsActivity.modelDir;
+            String detModelName = "ch_PP-OCRv2_det_infer";
             // String detModelName = "ch_ppocr_mobile_v2.0_det_infer";
             String clsModelName = "ch_ppocr_mobile_v2.0_cls_infer";
             // String recModelName = "ch_ppocr_mobile_v2.0_rec_infer";
-            String recModelName = "ch_PP-OCRv3_rec_infer";
-            // String recModelName = "ch_PP-OCRv2_rec_infer";
+            String recModelName = "ch_PP-OCRv2_rec_infer";
             String realDetModelDir = realModelDir + "/" + detModelName;
             String realClsModelDir = realModelDir + "/" + clsModelName;
             String realRecModelDir = realModelDir + "/" + recModelName;
@@ -236,16 +234,13 @@ public void checkAndUpdateSettings() {
             RuntimeOption detOption = new RuntimeOption();
             RuntimeOption clsOption = new RuntimeOption();
             RuntimeOption recOption = new RuntimeOption();
-            detOption.setCpuThreadNum(OcrSettingsActivity.cpuThreadNum);
-            clsOption.setCpuThreadNum(OcrSettingsActivity.cpuThreadNum);
-            recOption.setCpuThreadNum(OcrSettingsActivity.cpuThreadNum);
-            detOption.setLitePowerMode(OcrSettingsActivity.cpuPowerMode);
-            clsOption.setLitePowerMode(OcrSettingsActivity.cpuPowerMode);
-            recOption.setLitePowerMode(OcrSettingsActivity.cpuPowerMode);
-            detOption.enableRecordTimeOfRuntime();
-            clsOption.enableRecordTimeOfRuntime();
-            recOption.enableRecordTimeOfRuntime();
-            if (Boolean.parseBoolean(OcrSettingsActivity.enableLiteFp16)) {
+            detOption.setCpuThreadNum(SettingsActivity.cpuThreadNum);
+            clsOption.setCpuThreadNum(SettingsActivity.cpuThreadNum);
+            recOption.setCpuThreadNum(SettingsActivity.cpuThreadNum);
+            detOption.setLitePowerMode(SettingsActivity.cpuPowerMode);
+            clsOption.setLitePowerMode(SettingsActivity.cpuPowerMode);
+            recOption.setLitePowerMode(SettingsActivity.cpuPowerMode);
+            if (Boolean.parseBoolean(SettingsActivity.enableLiteFp16)) {
                 detOption.enableLiteFp16();
                 clsOption.enableLiteFp16();
                 recOption.enableLiteFp16();

java/android/app/src/main/res/layout/detection_camera_page.xml

@@ -52,7 +52,6 @@
                     android:text="@string/action_bar_realtime"
                     android:textAlignment="center" />
             </LinearLayout>
-        </com.baidu.paddle.fastdeploy.app.ui.layout.ActionBarLayout>
 
         <!--        实时-->
         <com.baidu.paddle.fastdeploy.app.ui.view.CameraSurfaceView

java/android/app/src/main/res/values/strings.xml

@@ -26,7 +26,7 @@
     <string name="OCR_REC_LABEL_DEFAULT">labels/ppocr_keys_v1.txt</string>
     <!-- Other resources values-->
     <string name="action_bar_take_photo">拍照识别</string>
-    <string name="action_bar_realtime">FD 实时识别</string>
+    <string name="action_bar_realtime">EasyEdge 实时识别</string>
     <string name="action_bar_back">&lt;</string>
     <string name="action_bar_model_name">模型名称</string>
     <string name="result_label">识别结果</string>