[FlyCV] optimize the integration of FlyCV (PaddlePaddle#492)

* [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android
Xiue233 · Nov 4, 2022 · a36d49a · a36d49a
1 parent 5b1fecd
commit a36d49a
Show file tree

Hide file tree

Showing 56 changed files with 667 additions and 484 deletions.
diff --git a/.gitignore b/.gitignore
@@ -38,3 +38,5 @@ coverage
 *.local
 yalc.*
 .yalc
+examples/vision/collect_quantize_cc.sh
+examples/vision/tests_quantize
diff --git a/cmake/gflags.cmake b/cmake/gflags.cmake
@@ -38,36 +38,71 @@ ENDIF(WIN32)
 
 INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})
 
-ExternalProject_Add(
-    extern_gflags
-    ${EXTERNAL_PROJECT_LOG_ARGS}
-    ${SHALLOW_CLONE}
-    GIT_REPOSITORY  ${GFLAGS_REPOSITORY}
-    GIT_TAG         ${GFLAGS_TAG}
-    PREFIX          ${GFLAGS_PREFIX_DIR}
-    UPDATE_COMMAND  ""
-    BUILD_COMMAND   ${BUILD_COMMAND}
-    INSTALL_COMMAND ${INSTALL_COMMAND}
-    CMAKE_ARGS      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
-                    -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
-                    -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
-                    -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
-                    -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
-                    -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
-                    -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
-                    -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
-                    -DBUILD_STATIC_LIBS=ON
-                    -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
-                    -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-                    -DBUILD_TESTING=OFF
-                    -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
-                    ${EXTERNAL_OPTIONAL_ARGS}
-    CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
-                     -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
-                     -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
-    BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
-)
-
+if(ANDROID)
+  ExternalProject_Add(
+      extern_gflags
+      ${EXTERNAL_PROJECT_LOG_ARGS}
+      ${SHALLOW_CLONE}
+      GIT_REPOSITORY  ${GFLAGS_REPOSITORY}
+      GIT_TAG         ${GFLAGS_TAG}
+      PREFIX          ${GFLAGS_PREFIX_DIR}
+      UPDATE_COMMAND  ""
+      BUILD_COMMAND   ${BUILD_COMMAND}
+      INSTALL_COMMAND ${INSTALL_COMMAND}
+      CMAKE_ARGS      -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}
+                      -DANDROID_ABI=${ANDROID_ABI}
+                      -DANDROID_NDK=${ANDROID_NDK}
+                      -DANDROID_PLATFORM=${ANDROID_PLATFORM}
+                      -DANDROID_STL=c++_static
+                      -DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN}
+                      -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+                      -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
+                      -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+                      -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+                      -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
+                      -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
+                      -DBUILD_STATIC_LIBS=ON
+                      -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
+                      -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                      -DBUILD_TESTING=OFF
+                      -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+                      ${EXTERNAL_OPTIONAL_ARGS}
+     CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
+                      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+                      -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}                
+      BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
+  )
+else()
+  ExternalProject_Add(
+      extern_gflags
+      ${EXTERNAL_PROJECT_LOG_ARGS}
+      ${SHALLOW_CLONE}
+      GIT_REPOSITORY  ${GFLAGS_REPOSITORY}
+      GIT_TAG         ${GFLAGS_TAG}
+      PREFIX          ${GFLAGS_PREFIX_DIR}
+      UPDATE_COMMAND  ""
+      BUILD_COMMAND   ${BUILD_COMMAND}
+      INSTALL_COMMAND ${INSTALL_COMMAND}
+      CMAKE_ARGS      -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER}
+                      -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER}
+                      -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}
+                      -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE}
+                      -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG}
+                      -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS}
+                      -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG}
+                      -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE}
+                      -DBUILD_STATIC_LIBS=ON
+                      -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR}
+                      -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+                      -DBUILD_TESTING=OFF
+                      -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+                      ${EXTERNAL_OPTIONAL_ARGS}
+      CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
+                      -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
+                      -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+      BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
+  )
+endif()
 ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL)
 SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES})
 ADD_DEPENDENCIES(gflags extern_gflags)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
@@ -37,12 +37,6 @@ function(config_fastdeploy_executable_link_flags TARGET_NAME)
   endif()
 endfunction()
 
-# Usage: add_fastdeploy_executable_cc_files(xxx_var vision detection)
-function(add_fastdeploy_executable_cc_files CC_FILES_VAR FIELD SUB_FIELD)
-  file(GLOB_RECURSE _EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/${FIELD}/${SUB_FIELD}/*/cpp/*.cc)
-  set(${CC_FILES_VAR} ${_EXAMPLE_SRCS} PARENT_SCOPE)
-endfunction()
-
 set(EXAMPLES_NUM 0)
 function(add_fastdeploy_executable FIELD CC_FILE)
   # temp target name/file var in function scope
@@ -55,7 +49,11 @@ function(add_fastdeploy_executable FIELD CC_FILE)
     add_executable(${TEMP_TARGET_NAME} ${TEMP_TARGET_FILE})
     target_link_libraries(${TEMP_TARGET_NAME} PUBLIC fastdeploy)
     if(TARGET gflags)
-      target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags pthread)
+      if(NOT ANDROID)
+        target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags pthread)
+      else()
+        target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags)
+      endif()
     endif()
     config_fastdeploy_executable_link_flags(${TEMP_TARGET_NAME})
     math(EXPR _EXAMPLES_NUM "${EXAMPLES_NUM} + 1")
@@ -78,22 +76,13 @@ if(BUILD_EXAMPLES AND ENABLE_VISION)
   if(EXISTS ${PROJECT_SOURCE_DIR}/examples/vision)
     message(STATUS "")
     message(STATUS "*************FastDeploy Vision Examples Summary**********")
-    set(ALL_VISION_SUD_FIELDS classification
-                              detection
-                              facedet
-                              faceid
-                              keypointdetection
-                              matting
-                              ocr
-                              segmentation)
-    if(NOT ANDROID)
-      list(APPEND ALL_VISION_SUD_FIELDS tracking)
+    file(GLOB_RECURSE ALL_VISION_EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/*/*/cpp/*.cc)
+    if(ANDROID)
+      file(GLOB_RECURSE TRACKING_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/tracking/*/cpp/*.cc)
+      list(REMOVE_ITEM ALL_VISION_EXAMPLE_SRCS ${TRACKING_SRCS})
     endif()
-    foreach(_SUB_FIELD ${ALL_VISION_SUD_FIELDS})
-      add_fastdeploy_executable_cc_files(_SUB_CC_FILES vision ${_SUB_FIELD})
-      foreach(_CC_FILE ${_SUB_CC_FILES})
-        add_fastdeploy_executable(vision ${_CC_FILE})
-      endforeach()
+    foreach(_CC_FILE ${ALL_VISION_EXAMPLE_SRCS})
+      add_fastdeploy_executable(vision ${_CC_FILE})
     endforeach()
     message(STATUS "  [FastDeploy Executable Path]        : ${EXECUTABLE_OUTPUT_PATH}")
   endif()

diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "fastdeploy/vision/classification/ppcls/model.h"
+
 #include "fastdeploy/vision/utils/utils.h"
 #include "yaml-cpp/yaml.h"
 
@@ -108,7 +109,7 @@ bool PaddleClasModel::Preprocess(Mat* mat, FDTensor* output) {
   int height = mat->Height();
   output->name = InputInfoOfRuntime(0).name;
   output->SetExternalData({1, channel, height, width}, FDDataType::FP32,
-                          mat->GetOpenCVMat()->ptr());
+                          mat->Data());
   return true;
 }
 

diff --git a/fastdeploy/vision/common/processors/base.cc b/fastdeploy/vision/common/processors/base.cc
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "fastdeploy/vision/common/processors/base.h"
+
 #include "fastdeploy/utils/utils.h"
 
 namespace fastdeploy {
@@ -21,27 +22,18 @@ namespace vision {
 ProcLib Processor::default_lib = ProcLib::DEFAULT;
 
 bool Processor::operator()(Mat* mat, ProcLib lib) {
-  // if default_lib is set
-  // then use default_lib
   ProcLib target = lib;
-  if (default_lib != ProcLib::DEFAULT) {
+  if (lib == ProcLib::DEFAULT) {
     target = default_lib;
   }
-
   if (target == ProcLib::FLYCV) {
 #ifdef ENABLE_FLYCV
-    if (mat->mat_type != ProcLib::FLYCV) {
-      if (mat->layout != Layout::HWC) {
-        FDERROR << "Cannot convert cv::Mat to fcv::Mat while layout is not HWC." << std::endl;
-      }
-      fcv::Mat fcv_mat = ConvertOpenCVMatToFalconCV(*(mat->GetOpenCVMat()));
-      mat->SetMat(fcv_mat);
-    }
-    return ImplByFalconCV(mat);
+    return ImplByFlyCV(mat);
 #else
-    FDASSERT(false, "FastDeploy didn't compile with FalconCV.");
+    FDASSERT(false, "FastDeploy didn't compile with FlyCV.");
 #endif
   }
+  // DEFAULT & OPENCV
   return ImplByOpenCV(mat);
 }
 
@@ -52,7 +44,7 @@ void EnableFlyCV() {
          << Processor::default_lib << std::endl;
 #else
   FDWARNING << "FastDeploy didn't compile with FlyCV, "
-                "will fallback to use OpenCV instead."
+               "will fallback to use OpenCV instead."
             << std::endl;
 #endif
 }
@@ -63,5 +55,100 @@ void DisableFlyCV() {
          << Processor::default_lib << std::endl;
 }
 
+cv::Mat CreateOpenCVMatFromTensor(const FDTensor& tensor) {
+  FDDataType type = tensor.dtype;
+  FDASSERT(tensor.shape.size() == 3,
+           "When create FD Mat from tensor, tensor shape should be 3-Dim, HWC "
+           "layout");
+  int64_t height = tensor.shape[0];
+  int64_t width = tensor.shape[1];
+  int64_t channel = tensor.shape[2];
+  cv::Mat ocv_mat;
+  // reference to outside FDTensor, zero copy
+  switch (type) {
+    case FDDataType::UINT8:
+      ocv_mat = cv::Mat(height, width, CV_8UC(channel),
+                        const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::INT8:
+      ocv_mat = cv::Mat(height, width, CV_8SC(channel),
+                        const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::INT16:
+      ocv_mat = cv::Mat(height, width, CV_16SC(channel),
+                        const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::INT32:
+      ocv_mat = cv::Mat(height, width, CV_32SC(channel),
+                        const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::FP32:
+      ocv_mat = cv::Mat(height, width, CV_32FC(channel),
+                        const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::FP64:
+      ocv_mat = cv::Mat(height, width, CV_64FC(channel),
+                        const_cast<void*>(tensor.Data()));
+      break;
+    default:
+      FDASSERT(false,
+               "Tensor type %d is not supported While calling "
+               "CreateFDMatFromTensor.",
+               type);
+      break;
+  }
+  return ocv_mat;
+}
+
+#ifdef ENABLE_FLYCV
+fcv::Mat CreateFlyCVMatFromTensor(const FDTensor& tensor) {
+  FDDataType type = tensor.dtype;
+  FDASSERT(tensor.shape.size() == 3,
+           "When create FD Mat from tensor, tensor shape should be 3-Dim, HWC "
+           "layout");
+  int64_t height = tensor.shape[0];
+  int64_t width = tensor.shape[1];
+  int64_t channel = tensor.shape[2];
+  fcv::Mat fcv_mat;
+  auto fcv_type = CreateFlyCVDataType(type, static_cast<int>(channel));
+  switch (type) {
+    case FDDataType::UINT8:
+      fcv_mat =
+        fcv::Mat(width, height, fcv_type, const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::FP32:
+      fcv_mat =
+        fcv::Mat(width, height, fcv_type, const_cast<void*>(tensor.Data()));
+      break;
+    case FDDataType::FP64:
+      fcv_mat =
+        fcv::Mat(width, height, fcv_type, const_cast<void*>(tensor.Data()));
+    break;
+    default:
+      FDASSERT(false,
+              "Tensor type %d is not supported While calling "
+              "CreateFDMatFromTensor.",
+               type);
+    break;
+  }
+  return fcv_mat;
+}
+#endif
+
+Mat CreateFDMatFromTensor(const FDTensor& tensor) {
+  if (Processor::default_lib == ProcLib::FLYCV) {
+#ifdef ENABLE_FLYCV
+    fcv::Mat fcv_mat = CreateFlyCVMatFromTensor(tensor);
+    Mat mat = Mat(fcv_mat);
+    return mat;
+#else
+    FDASSERT(false, "FastDeploy didn't compiled with FlyCV!");
+#endif
+  }
+  cv::Mat ocv_mat = CreateOpenCVMatFromTensor(tensor);
+  Mat mat = Mat(ocv_mat);
+  return mat;
+}
+
 }  // namespace vision
 }  // namespace fastdeploy
diff --git a/fastdeploy/vision/common/processors/base.h b/fastdeploy/vision/common/processors/base.h
@@ -22,7 +22,9 @@
 namespace fastdeploy {
 namespace vision {
 
-/*! @brief Enable using FlyCV to process image while deploy vision models. Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will fallback to using OpenCV in other platform
+/*! @brief Enable using FlyCV to process image while deploy vision models.
+ * Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will
+ * fallback to using OpenCV in other platform
  */
 FASTDEPLOY_DECL void EnableFlyCV();
 
@@ -41,16 +43,19 @@ class FASTDEPLOY_DECL Processor {
 
   virtual bool ImplByOpenCV(Mat* mat) = 0;
 
-  virtual bool ImplByFalconCV(Mat* mat) {
-    FDASSERT(false,
-             "%s is not implemented with FalconCV, please use OpenCV instead.",
-             Name().c_str());
-    return false;
+  virtual bool ImplByFlyCV(Mat* mat) {
+    return ImplByOpenCV(mat);
   }
 
-  virtual bool operator()(Mat* mat,
-                          ProcLib lib = ProcLib::OPENCV);
+  virtual bool operator()(Mat* mat, ProcLib lib = ProcLib::DEFAULT);
 };
 
+// Create OpenCV/FlyCV/FD Mat from FD Tensor
+cv::Mat CreateOpenCVMatFromTensor(const FDTensor& tensor);
+#ifdef ENABLE_FLYCV
+fcv::Mat CreateFlyCVMatFromTensor(const FDTensor& tensor);
+#endif
+Mat CreateFDMatFromTensor(const FDTensor& tensor);
+
 }  // namespace vision
 }  // namespace fastdeploy
-Original file line number
+Diff line change
@@ Expand Up / @@ -38,3 +38,5 @@ coverage @@
     *.local
     yalc.*
     .yalc
+    examples/vision/collect_quantize_cc.sh
+    examples/vision/tests_quantize