From a36d49a973a303a2d0524971044745a807fe7d36 Mon Sep 17 00:00:00 2001 From: DefTruth <31974251+DefTruth@users.noreply.github.com> Date: Fri, 4 Nov 2022 09:19:03 +0800 Subject: [PATCH] [FlyCV] optimize the integration of FlyCV (#492) * [Backend] fix lite backend save model error * [Backend] fixed typos * [FlyCV] optimize the integration of FlyCV * [cmake] close some tests options * [cmake] close some test option * [FlyCV] remove un-need warnings * [FlyCV] remove un-need GetMat method * [FlyCV] optimize FlyCV codes * [cmake] remove un-need cmake function in examples/CMakelists * [cmake] support gflags for Android --- .gitignore | 2 + cmake/gflags.cmake | 95 ++++++++++----- examples/CMakeLists.txt | 33 ++--- .../vision/classification/ppcls/model.cc | 3 +- fastdeploy/vision/common/processors/base.cc | 115 +++++++++++++++--- fastdeploy/vision/common/processors/base.h | 21 ++-- fastdeploy/vision/common/processors/cast.cc | 10 +- fastdeploy/vision/common/processors/cast.h | 8 +- .../vision/common/processors/center_crop.cc | 8 +- .../vision/common/processors/center_crop.h | 4 +- .../common/processors/color_space_convert.cc | 14 ++- .../common/processors/color_space_convert.h | 8 +- .../vision/common/processors/convert.cc | 11 +- fastdeploy/vision/common/processors/convert.h | 4 +- fastdeploy/vision/common/processors/crop.cc | 4 +- fastdeploy/vision/common/processors/crop.h | 4 +- .../vision/common/processors/hwc2chw.cc | 18 +-- fastdeploy/vision/common/processors/hwc2chw.h | 4 +- .../vision/common/processors/letter_box.h | 6 +- .../common/processors/limit_by_stride.cc | 8 +- .../common/processors/limit_by_stride.h | 4 +- .../vision/common/processors/limit_long.cc | 12 +- .../vision/common/processors/limit_long.h | 4 +- .../vision/common/processors/limit_short.cc | 9 +- .../vision/common/processors/limit_short.h | 4 +- fastdeploy/vision/common/processors/mat.cc | 114 +++++------------ fastdeploy/vision/common/processors/mat.h | 79 +++++++----- .../vision/common/processors/normalize.cc | 20 +-- .../vision/common/processors/normalize.h | 12 +- .../processors/normalize_and_permute.cc | 36 +++--- .../common/processors/normalize_and_permute.h | 11 +- fastdeploy/vision/common/processors/pad.cc | 13 +- fastdeploy/vision/common/processors/pad.h | 4 +- .../vision/common/processors/pad_to_size.cc | 9 +- .../vision/common/processors/pad_to_size.h | 4 +- fastdeploy/vision/common/processors/resize.cc | 13 +- fastdeploy/vision/common/processors/resize.h | 4 +- .../vision/common/processors/resize_by_long.h | 2 +- .../common/processors/resize_by_short.cc | 6 +- .../common/processors/resize_by_short.h | 4 +- .../vision/common/processors/stride_pad.cc | 7 +- .../vision/common/processors/stride_pad.h | 4 +- .../vision/common/processors/transform.h | 52 ++++---- fastdeploy/vision/common/processors/utils.cc | 67 ++++++++-- fastdeploy/vision/common/processors/utils.h | 14 ++- .../vision/common/processors/warp_affine.h | 15 +-- fastdeploy/vision/detection/contrib/yolov5.cc | 49 ++++---- fastdeploy/vision/detection/contrib/yolov6.cc | 30 +++-- fastdeploy/vision/detection/contrib/yolov7.cc | 30 +++-- fastdeploy/vision/matting/contrib/modnet.cc | 7 +- fastdeploy/vision/matting/contrib/rvm.cc | 33 ++--- .../vision/matting/ppmatting/ppmatting.cc | 14 ++- fastdeploy/vision/segmentation/ppseg/model.cc | 31 +++-- .../vision/tracking/pptracking/trajectory.h | 9 +- fastdeploy/vision/utils/crop_image.cc | 14 +-- fastdeploy/vision/utils/utils.h | 31 ++--- 56 files changed, 667 insertions(+), 484 deletions(-) mode change 100755 => 100644 fastdeploy/vision/common/processors/letter_box.h mode change 100755 => 100644 fastdeploy/vision/matting/contrib/modnet.cc mode change 100755 => 100644 fastdeploy/vision/matting/contrib/rvm.cc mode change 100755 => 100644 fastdeploy/vision/matting/ppmatting/ppmatting.cc mode change 100755 => 100644 fastdeploy/vision/segmentation/ppseg/model.cc diff --git a/.gitignore b/.gitignore index db38f7705d..2edbc3103c 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ coverage *.local yalc.* .yalc +examples/vision/collect_quantize_cc.sh +examples/vision/tests_quantize \ No newline at end of file diff --git a/cmake/gflags.cmake b/cmake/gflags.cmake index 3402383cb7..08d5e49c81 100644 --- a/cmake/gflags.cmake +++ b/cmake/gflags.cmake @@ -38,36 +38,71 @@ ENDIF(WIN32) INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) -ExternalProject_Add( - extern_gflags - ${EXTERNAL_PROJECT_LOG_ARGS} - ${SHALLOW_CLONE} - GIT_REPOSITORY ${GFLAGS_REPOSITORY} - GIT_TAG ${GFLAGS_TAG} - PREFIX ${GFLAGS_PREFIX_DIR} - UPDATE_COMMAND "" - BUILD_COMMAND ${BUILD_COMMAND} - INSTALL_COMMAND ${INSTALL_COMMAND} - CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} - -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} - -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} - -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} - -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} - -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} - -DBUILD_STATIC_LIBS=ON - -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} - -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DBUILD_TESTING=OFF - -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} - ${EXTERNAL_OPTIONAL_ARGS} - CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR} - -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON - -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} - BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES} -) - +if(ANDROID) + ExternalProject_Add( + extern_gflags + ${EXTERNAL_PROJECT_LOG_ARGS} + ${SHALLOW_CLONE} + GIT_REPOSITORY ${GFLAGS_REPOSITORY} + GIT_TAG ${GFLAGS_TAG} + PREFIX ${GFLAGS_PREFIX_DIR} + UPDATE_COMMAND "" + BUILD_COMMAND ${BUILD_COMMAND} + INSTALL_COMMAND ${INSTALL_COMMAND} + CMAKE_ARGS -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} + -DANDROID_ABI=${ANDROID_ABI} + -DANDROID_NDK=${ANDROID_NDK} + -DANDROID_PLATFORM=${ANDROID_PLATFORM} + -DANDROID_STL=c++_static + -DANDROID_TOOLCHAIN=${ANDROID_TOOLCHAIN} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} + -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} + -DBUILD_STATIC_LIBS=ON + -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_TESTING=OFF + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + ${EXTERNAL_OPTIONAL_ARGS} + CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES} + ) +else() + ExternalProject_Add( + extern_gflags + ${EXTERNAL_PROJECT_LOG_ARGS} + ${SHALLOW_CLONE} + GIT_REPOSITORY ${GFLAGS_REPOSITORY} + GIT_TAG ${GFLAGS_TAG} + PREFIX ${GFLAGS_PREFIX_DIR} + UPDATE_COMMAND "" + BUILD_COMMAND ${BUILD_COMMAND} + INSTALL_COMMAND ${INSTALL_COMMAND} + CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} + -DCMAKE_CXX_FLAGS_RELEASE=${CMAKE_CXX_FLAGS_RELEASE} + -DCMAKE_CXX_FLAGS_DEBUG=${CMAKE_CXX_FLAGS_DEBUG} + -DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} + -DCMAKE_C_FLAGS_DEBUG=${CMAKE_C_FLAGS_DEBUG} + -DCMAKE_C_FLAGS_RELEASE=${CMAKE_C_FLAGS_RELEASE} + -DBUILD_STATIC_LIBS=ON + -DCMAKE_INSTALL_PREFIX=${GFLAGS_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DBUILD_TESTING=OFF + -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + ${EXTERNAL_OPTIONAL_ARGS} + CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR} + -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON + -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE} + BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES} + ) +endif() ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES}) ADD_DEPENDENCIES(gflags extern_gflags) diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 1f14fa8897..e0d99a30ab 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -37,12 +37,6 @@ function(config_fastdeploy_executable_link_flags TARGET_NAME) endif() endfunction() -# Usage: add_fastdeploy_executable_cc_files(xxx_var vision detection) -function(add_fastdeploy_executable_cc_files CC_FILES_VAR FIELD SUB_FIELD) - file(GLOB_RECURSE _EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/${FIELD}/${SUB_FIELD}/*/cpp/*.cc) - set(${CC_FILES_VAR} ${_EXAMPLE_SRCS} PARENT_SCOPE) -endfunction() - set(EXAMPLES_NUM 0) function(add_fastdeploy_executable FIELD CC_FILE) # temp target name/file var in function scope @@ -55,7 +49,11 @@ function(add_fastdeploy_executable FIELD CC_FILE) add_executable(${TEMP_TARGET_NAME} ${TEMP_TARGET_FILE}) target_link_libraries(${TEMP_TARGET_NAME} PUBLIC fastdeploy) if(TARGET gflags) - target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags pthread) + if(NOT ANDROID) + target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags pthread) + else() + target_link_libraries(${TEMP_TARGET_NAME} PRIVATE gflags) + endif() endif() config_fastdeploy_executable_link_flags(${TEMP_TARGET_NAME}) math(EXPR _EXAMPLES_NUM "${EXAMPLES_NUM} + 1") @@ -78,22 +76,13 @@ if(BUILD_EXAMPLES AND ENABLE_VISION) if(EXISTS ${PROJECT_SOURCE_DIR}/examples/vision) message(STATUS "") message(STATUS "*************FastDeploy Vision Examples Summary**********") - set(ALL_VISION_SUD_FIELDS classification - detection - facedet - faceid - keypointdetection - matting - ocr - segmentation) - if(NOT ANDROID) - list(APPEND ALL_VISION_SUD_FIELDS tracking) + file(GLOB_RECURSE ALL_VISION_EXAMPLE_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/*/*/cpp/*.cc) + if(ANDROID) + file(GLOB_RECURSE TRACKING_SRCS ${PROJECT_SOURCE_DIR}/examples/vision/tracking/*/cpp/*.cc) + list(REMOVE_ITEM ALL_VISION_EXAMPLE_SRCS ${TRACKING_SRCS}) endif() - foreach(_SUB_FIELD ${ALL_VISION_SUD_FIELDS}) - add_fastdeploy_executable_cc_files(_SUB_CC_FILES vision ${_SUB_FIELD}) - foreach(_CC_FILE ${_SUB_CC_FILES}) - add_fastdeploy_executable(vision ${_CC_FILE}) - endforeach() + foreach(_CC_FILE ${ALL_VISION_EXAMPLE_SRCS}) + add_fastdeploy_executable(vision ${_CC_FILE}) endforeach() message(STATUS " [FastDeploy Executable Path] : ${EXECUTABLE_OUTPUT_PATH}") endif() diff --git a/fastdeploy/vision/classification/ppcls/model.cc b/fastdeploy/vision/classification/ppcls/model.cc index 57e3541117..3eed25c6cd 100644 --- a/fastdeploy/vision/classification/ppcls/model.cc +++ b/fastdeploy/vision/classification/ppcls/model.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/classification/ppcls/model.h" + #include "fastdeploy/vision/utils/utils.h" #include "yaml-cpp/yaml.h" @@ -108,7 +109,7 @@ bool PaddleClasModel::Preprocess(Mat* mat, FDTensor* output) { int height = mat->Height(); output->name = InputInfoOfRuntime(0).name; output->SetExternalData({1, channel, height, width}, FDDataType::FP32, - mat->GetOpenCVMat()->ptr()); + mat->Data()); return true; } diff --git a/fastdeploy/vision/common/processors/base.cc b/fastdeploy/vision/common/processors/base.cc index 4e26297020..f7831ae638 100644 --- a/fastdeploy/vision/common/processors/base.cc +++ b/fastdeploy/vision/common/processors/base.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/common/processors/base.h" + #include "fastdeploy/utils/utils.h" namespace fastdeploy { @@ -21,27 +22,18 @@ namespace vision { ProcLib Processor::default_lib = ProcLib::DEFAULT; bool Processor::operator()(Mat* mat, ProcLib lib) { - // if default_lib is set - // then use default_lib ProcLib target = lib; - if (default_lib != ProcLib::DEFAULT) { + if (lib == ProcLib::DEFAULT) { target = default_lib; } - if (target == ProcLib::FLYCV) { #ifdef ENABLE_FLYCV - if (mat->mat_type != ProcLib::FLYCV) { - if (mat->layout != Layout::HWC) { - FDERROR << "Cannot convert cv::Mat to fcv::Mat while layout is not HWC." << std::endl; - } - fcv::Mat fcv_mat = ConvertOpenCVMatToFalconCV(*(mat->GetOpenCVMat())); - mat->SetMat(fcv_mat); - } - return ImplByFalconCV(mat); + return ImplByFlyCV(mat); #else - FDASSERT(false, "FastDeploy didn't compile with FalconCV."); + FDASSERT(false, "FastDeploy didn't compile with FlyCV."); #endif } + // DEFAULT & OPENCV return ImplByOpenCV(mat); } @@ -52,7 +44,7 @@ void EnableFlyCV() { << Processor::default_lib << std::endl; #else FDWARNING << "FastDeploy didn't compile with FlyCV, " - "will fallback to use OpenCV instead." + "will fallback to use OpenCV instead." << std::endl; #endif } @@ -63,5 +55,100 @@ void DisableFlyCV() { << Processor::default_lib << std::endl; } +cv::Mat CreateOpenCVMatFromTensor(const FDTensor& tensor) { + FDDataType type = tensor.dtype; + FDASSERT(tensor.shape.size() == 3, + "When create FD Mat from tensor, tensor shape should be 3-Dim, HWC " + "layout"); + int64_t height = tensor.shape[0]; + int64_t width = tensor.shape[1]; + int64_t channel = tensor.shape[2]; + cv::Mat ocv_mat; + // reference to outside FDTensor, zero copy + switch (type) { + case FDDataType::UINT8: + ocv_mat = cv::Mat(height, width, CV_8UC(channel), + const_cast(tensor.Data())); + break; + case FDDataType::INT8: + ocv_mat = cv::Mat(height, width, CV_8SC(channel), + const_cast(tensor.Data())); + break; + case FDDataType::INT16: + ocv_mat = cv::Mat(height, width, CV_16SC(channel), + const_cast(tensor.Data())); + break; + case FDDataType::INT32: + ocv_mat = cv::Mat(height, width, CV_32SC(channel), + const_cast(tensor.Data())); + break; + case FDDataType::FP32: + ocv_mat = cv::Mat(height, width, CV_32FC(channel), + const_cast(tensor.Data())); + break; + case FDDataType::FP64: + ocv_mat = cv::Mat(height, width, CV_64FC(channel), + const_cast(tensor.Data())); + break; + default: + FDASSERT(false, + "Tensor type %d is not supported While calling " + "CreateFDMatFromTensor.", + type); + break; + } + return ocv_mat; +} + +#ifdef ENABLE_FLYCV +fcv::Mat CreateFlyCVMatFromTensor(const FDTensor& tensor) { + FDDataType type = tensor.dtype; + FDASSERT(tensor.shape.size() == 3, + "When create FD Mat from tensor, tensor shape should be 3-Dim, HWC " + "layout"); + int64_t height = tensor.shape[0]; + int64_t width = tensor.shape[1]; + int64_t channel = tensor.shape[2]; + fcv::Mat fcv_mat; + auto fcv_type = CreateFlyCVDataType(type, static_cast(channel)); + switch (type) { + case FDDataType::UINT8: + fcv_mat = + fcv::Mat(width, height, fcv_type, const_cast(tensor.Data())); + break; + case FDDataType::FP32: + fcv_mat = + fcv::Mat(width, height, fcv_type, const_cast(tensor.Data())); + break; + case FDDataType::FP64: + fcv_mat = + fcv::Mat(width, height, fcv_type, const_cast(tensor.Data())); + break; + default: + FDASSERT(false, + "Tensor type %d is not supported While calling " + "CreateFDMatFromTensor.", + type); + break; + } + return fcv_mat; +} +#endif + +Mat CreateFDMatFromTensor(const FDTensor& tensor) { + if (Processor::default_lib == ProcLib::FLYCV) { +#ifdef ENABLE_FLYCV + fcv::Mat fcv_mat = CreateFlyCVMatFromTensor(tensor); + Mat mat = Mat(fcv_mat); + return mat; +#else + FDASSERT(false, "FastDeploy didn't compiled with FlyCV!"); +#endif + } + cv::Mat ocv_mat = CreateOpenCVMatFromTensor(tensor); + Mat mat = Mat(ocv_mat); + return mat; +} + } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/base.h b/fastdeploy/vision/common/processors/base.h index d985f348fc..bfd1e00856 100644 --- a/fastdeploy/vision/common/processors/base.h +++ b/fastdeploy/vision/common/processors/base.h @@ -22,7 +22,9 @@ namespace fastdeploy { namespace vision { -/*! @brief Enable using FlyCV to process image while deploy vision models. Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will fallback to using OpenCV in other platform +/*! @brief Enable using FlyCV to process image while deploy vision models. + * Currently, FlyCV in only available on ARM(Linux aarch64/Android), so will + * fallback to using OpenCV in other platform */ FASTDEPLOY_DECL void EnableFlyCV(); @@ -41,16 +43,19 @@ class FASTDEPLOY_DECL Processor { virtual bool ImplByOpenCV(Mat* mat) = 0; - virtual bool ImplByFalconCV(Mat* mat) { - FDASSERT(false, - "%s is not implemented with FalconCV, please use OpenCV instead.", - Name().c_str()); - return false; + virtual bool ImplByFlyCV(Mat* mat) { + return ImplByOpenCV(mat); } - virtual bool operator()(Mat* mat, - ProcLib lib = ProcLib::OPENCV); + virtual bool operator()(Mat* mat, ProcLib lib = ProcLib::DEFAULT); }; +// Create OpenCV/FlyCV/FD Mat from FD Tensor +cv::Mat CreateOpenCVMatFromTensor(const FDTensor& tensor); +#ifdef ENABLE_FLYCV +fcv::Mat CreateFlyCVMatFromTensor(const FDTensor& tensor); +#endif +Mat CreateFDMatFromTensor(const FDTensor& tensor); + } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/cast.cc b/fastdeploy/vision/common/processors/cast.cc index 62aad3778f..0ca04a504e 100644 --- a/fastdeploy/vision/common/processors/cast.cc +++ b/fastdeploy/vision/common/processors/cast.cc @@ -36,8 +36,8 @@ bool Cast::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool Cast::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool Cast::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); if (dtype_ == "float" && mat->Type() == FDDataType::FP32) { return true; } @@ -46,18 +46,18 @@ bool Cast::ImplByFalconCV(Mat* mat) { } if (mat->layout != Layout::HWC) { FDERROR - << "While using Falcon to cast image, the image must be layout of HWC." + << "While using FlyCV to cast image, the image must be layout of HWC." << std::endl; return false; } if (dtype_ == "float") { fcv::Mat new_im; - auto fcv_type = CreateFalconCVDataType(FDDataType::FP32, im->channels()); + auto fcv_type = CreateFlyCVDataType(FDDataType::FP32, im->channels()); im->convert_to(new_im, fcv_type); mat->SetMat(new_im); } else if (dtype_ == "double") { fcv::Mat new_im; - auto fcv_type = CreateFalconCVDataType(FDDataType::FP64, im->channels()); + auto fcv_type = CreateFlyCVDataType(FDDataType::FP64, im->channels()); im->convert_to(new_im, fcv_type); mat->SetMat(new_im); } else { diff --git a/fastdeploy/vision/common/processors/cast.h b/fastdeploy/vision/common/processors/cast.h index a438fb9743..891ae334c8 100644 --- a/fastdeploy/vision/common/processors/cast.h +++ b/fastdeploy/vision/common/processors/cast.h @@ -24,15 +24,13 @@ class FASTDEPLOY_DECL Cast : public Processor { explicit Cast(const std::string& dtype = "float") : dtype_(dtype) {} bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "Cast"; } static bool Run(Mat* mat, const std::string& dtype, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); - std::string GetDtype() const { - return dtype_; - } + std::string GetDtype() const { return dtype_; } private: std::string dtype_; diff --git a/fastdeploy/vision/common/processors/center_crop.cc b/fastdeploy/vision/common/processors/center_crop.cc index 6aca3acd09..af7c744482 100644 --- a/fastdeploy/vision/common/processors/center_crop.cc +++ b/fastdeploy/vision/common/processors/center_crop.cc @@ -36,8 +36,8 @@ bool CenterCrop::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool CenterCrop::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool CenterCrop::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); int height = static_cast(im->height()); int width = static_cast(im->width()); if (height < height_ || width < width_) { @@ -62,5 +62,5 @@ bool CenterCrop::Run(Mat* mat, const int& width, const int& height, return c(mat, lib); } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/center_crop.h b/fastdeploy/vision/common/processors/center_crop.h index befeeb5a35..05f594249c 100644 --- a/fastdeploy/vision/common/processors/center_crop.h +++ b/fastdeploy/vision/common/processors/center_crop.h @@ -24,12 +24,12 @@ class FASTDEPLOY_DECL CenterCrop : public Processor { CenterCrop(int width, int height) : height_(height), width_(width) {} bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "CenterCrop"; } static bool Run(Mat* mat, const int& width, const int& height, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: int height_; diff --git a/fastdeploy/vision/common/processors/color_space_convert.cc b/fastdeploy/vision/common/processors/color_space_convert.cc index 15e8aa4265..4ccfb65100 100644 --- a/fastdeploy/vision/common/processors/color_space_convert.cc +++ b/fastdeploy/vision/common/processors/color_space_convert.cc @@ -25,10 +25,11 @@ bool BGR2RGB::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool BGR2RGB::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool BGR2RGB::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { - FDERROR << "[BGR2RGB] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl; + FDERROR << "[BGR2RGB] The channel of input image must be 3, but not it's " + << im->channels() << "." << std::endl; return false; } fcv::Mat new_im; @@ -47,10 +48,11 @@ bool RGB2BGR::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool RGB2BGR::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool RGB2BGR::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { - FDERROR << "[RGB2BGR] The channel of input image must be 3, but not it's " << im->channels() << "." << std::endl; + FDERROR << "[RGB2BGR] The channel of input image must be 3, but not it's " + << im->channels() << "." << std::endl; return false; } fcv::Mat new_im; diff --git a/fastdeploy/vision/common/processors/color_space_convert.h b/fastdeploy/vision/common/processors/color_space_convert.h index ad66acd4f3..e090bc62d5 100644 --- a/fastdeploy/vision/common/processors/color_space_convert.h +++ b/fastdeploy/vision/common/processors/color_space_convert.h @@ -23,22 +23,22 @@ class FASTDEPLOY_DECL BGR2RGB : public Processor { public: bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif virtual std::string Name() { return "BGR2RGB"; } - static bool Run(Mat* mat, ProcLib lib = ProcLib::OPENCV); + static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); }; class FASTDEPLOY_DECL RGB2BGR : public Processor { public: bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "RGB2BGR"; } - static bool Run(Mat* mat, ProcLib lib = ProcLib::OPENCV); + static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); }; } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/convert.cc b/fastdeploy/vision/common/processors/convert.cc index 01c0673701..c50d2aa6d4 100644 --- a/fastdeploy/vision/common/processors/convert.cc +++ b/fastdeploy/vision/common/processors/convert.cc @@ -40,17 +40,18 @@ bool Convert::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool Convert::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); - FDASSERT(im->channels() == 3, "Only support 3-channels image in FalconCV."); +bool Convert::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); + FDASSERT(im->channels() == 3, "Only support 3-channels image in FlyCV."); std::vector mean(3, 0); std::vector std(3, 0); for (size_t i = 0; i < 3; ++i) { - std[i] = 1.0 / alpha_[i]; + std[i] = 1.0 / alpha_[i]; mean[i] = -1 * beta_[i] * std[i]; } fcv::Mat new_im; - fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector(), new_im, true); + fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector(), + new_im, true); mat->SetMat(new_im); return true; } diff --git a/fastdeploy/vision/common/processors/convert.h b/fastdeploy/vision/common/processors/convert.h index 197316a52c..6f26c45282 100644 --- a/fastdeploy/vision/common/processors/convert.h +++ b/fastdeploy/vision/common/processors/convert.h @@ -24,7 +24,7 @@ class FASTDEPLOY_DECL Convert : public Processor { bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "Convert"; } @@ -32,7 +32,7 @@ class FASTDEPLOY_DECL Convert : public Processor { // The default behavior is the same as OpenCV's convertTo method. static bool Run(Mat* mat, const std::vector& alpha, const std::vector& beta, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: std::vector alpha_; diff --git a/fastdeploy/vision/common/processors/crop.cc b/fastdeploy/vision/common/processors/crop.cc index fc3a41ecba..ccb06f6539 100644 --- a/fastdeploy/vision/common/processors/crop.cc +++ b/fastdeploy/vision/common/processors/crop.cc @@ -37,8 +37,8 @@ bool Crop::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool Crop::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool Crop::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); int height = static_cast(im->height()); int width = static_cast(im->width()); if (height < height_ + offset_h_ || width < width_ + offset_w_) { diff --git a/fastdeploy/vision/common/processors/crop.h b/fastdeploy/vision/common/processors/crop.h index 276d7d6cf6..369edb610f 100644 --- a/fastdeploy/vision/common/processors/crop.h +++ b/fastdeploy/vision/common/processors/crop.h @@ -31,12 +31,12 @@ class FASTDEPLOY_DECL Crop : public Processor { bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "Crop"; } static bool Run(Mat* mat, int offset_w, int offset_h, int width, int height, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: int offset_w_; diff --git a/fastdeploy/vision/common/processors/hwc2chw.cc b/fastdeploy/vision/common/processors/hwc2chw.cc index 3af9b559d5..9db5c09ffd 100644 --- a/fastdeploy/vision/common/processors/hwc2chw.cc +++ b/fastdeploy/vision/common/processors/hwc2chw.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/common/processors/hwc2chw.h" + #include "fastdeploy/function/transpose.h" namespace fastdeploy { @@ -41,18 +42,21 @@ bool HWC2CHW::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool HWC2CHW::ImplByFalconCV(Mat* mat) { +bool HWC2CHW::ImplByFlyCV(Mat* mat) { if (mat->layout != Layout::HWC) { - FDERROR << "HWC2CHW: The input data is not Layout::HWC format!" << std::endl; + FDERROR << "HWC2CHW: The input data is not Layout::HWC format!" + << std::endl; return false; } if (mat->Type() != FDDataType::FP32) { - FDERROR << "HWC2CHW: Only support float data while use FalconCV, but now it's " << mat->Type() << "." << std::endl; + FDERROR << "HWC2CHW: Only support float data while use FlyCV, but now it's " + << mat->Type() << "." << std::endl; return false; } - fcv::Mat* im = mat->GetFalconCVMat(); + fcv::Mat* im = mat->GetFlyCVMat(); fcv::Mat new_im; - fcv::normalize_to_submean_to_reorder(*im, {0.0, 0.0, 0.0}, {1.0, 1.0, 1.0}, std::vector(), new_im, false); + fcv::normalize_to_submean_to_reorder(*im, {0.0, 0.0, 0.0}, {1.0, 1.0, 1.0}, + std::vector(), new_im, false); mat->SetMat(new_im); mat->layout = Layout::CHW; return true; @@ -64,5 +68,5 @@ bool HWC2CHW::Run(Mat* mat, ProcLib lib) { return h(mat, lib); } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/hwc2chw.h b/fastdeploy/vision/common/processors/hwc2chw.h index a21de7b4c3..535a1887b5 100644 --- a/fastdeploy/vision/common/processors/hwc2chw.h +++ b/fastdeploy/vision/common/processors/hwc2chw.h @@ -23,11 +23,11 @@ class FASTDEPLOY_DECL HWC2CHW : public Processor { public: bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "HWC2CHW"; } - static bool Run(Mat* mat, ProcLib lib = ProcLib::OPENCV); + static bool Run(Mat* mat, ProcLib lib = ProcLib::DEFAULT); }; } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/letter_box.h b/fastdeploy/vision/common/processors/letter_box.h old mode 100755 new mode 100644 index 5b99ada2ee..f69647e892 --- a/fastdeploy/vision/common/processors/letter_box.h +++ b/fastdeploy/vision/common/processors/letter_box.h @@ -23,8 +23,8 @@ class LetterBoxResize : public Processor { public: LetterBoxResize(const std::vector& target_size, const std::vector& color) { - target_size_ = target_size; - color_ = color; + target_size_ = target_size; + color_ = color; } bool ImplByOpenCV(Mat* mat); @@ -33,7 +33,7 @@ class LetterBoxResize : public Processor { static bool Run(Mat* mat, const std::vector& target_size, const std::vector& color, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: std::vector target_size_; diff --git a/fastdeploy/vision/common/processors/limit_by_stride.cc b/fastdeploy/vision/common/processors/limit_by_stride.cc index 85f3088b80..3efbc752f0 100644 --- a/fastdeploy/vision/common/processors/limit_by_stride.cc +++ b/fastdeploy/vision/common/processors/limit_by_stride.cc @@ -38,8 +38,8 @@ bool LimitByStride::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool LimitByStride::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool LimitByStride::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); int origin_w = im->width(); int origin_h = im->height(); int rw = origin_w - origin_w % stride_; @@ -59,16 +59,14 @@ bool LimitByStride::ImplByFalconCV(Mat* mat) { } else if (interp_ == 2) { interp_method = fcv::InterpolationType::INTER_CUBIC; } else { - FDERROR << "LimitByStride: Only support interp_ be 0/1/2 with FalconCV, but " + FDERROR << "LimitByStride: Only support interp_ be 0/1/2 with FlyCV, but " "now it's " << interp_ << "." << std::endl; return false; } fcv::Mat new_im; - FDERROR << "Before " << im->width() << " " << im->height() << std::endl; fcv::resize(*im, new_im, fcv::Size(rw, rh), 0, 0, interp_method); - FDERROR << "After " << new_im.width() << " " << new_im.height() << std::endl; mat->SetMat(new_im); mat->SetWidth(new_im.width()); mat->SetHeight(new_im.height()); diff --git a/fastdeploy/vision/common/processors/limit_by_stride.h b/fastdeploy/vision/common/processors/limit_by_stride.h index 465bc065d4..a3751df2e9 100644 --- a/fastdeploy/vision/common/processors/limit_by_stride.h +++ b/fastdeploy/vision/common/processors/limit_by_stride.h @@ -29,12 +29,12 @@ class FASTDEPLOY_DECL LimitByStride : public Processor { // Resize Mat* mat to make the size divisible by stride_. bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "LimitByStride"; } static bool Run(Mat* mat, int stride = 32, int interp = 1, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: int interp_; diff --git a/fastdeploy/vision/common/processors/limit_long.cc b/fastdeploy/vision/common/processors/limit_long.cc index 32eaad1923..7021f131b4 100644 --- a/fastdeploy/vision/common/processors/limit_long.cc +++ b/fastdeploy/vision/common/processors/limit_long.cc @@ -39,8 +39,8 @@ bool LimitLong::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool LimitLong::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool LimitLong::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); int origin_w = im->width(); int origin_h = im->height(); int im_size_max = std::max(origin_w, origin_h); @@ -64,16 +64,13 @@ bool LimitLong::ImplByFalconCV(Mat* mat) { } else if (interp_ == 2) { interp_method = fcv::InterpolationType::INTER_CUBIC; } else { - FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but " + FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but " "now it's " << interp_ << "." << std::endl; return false; } fcv::Mat new_im; - FDERROR << "origin " << im->width() << " " << im->height() << std::endl; - FDERROR << "scale " << scale << std::endl; fcv::resize(*im, new_im, fcv::Size(), scale, scale, interp_method); - FDERROR << "after " << new_im.width() << " " << new_im.height() << std::endl; mat->SetMat(new_im); mat->SetWidth(new_im.width()); mat->SetHeight(new_im.height()); @@ -82,7 +79,8 @@ bool LimitLong::ImplByFalconCV(Mat* mat) { } #endif -bool LimitLong::Run(Mat* mat, int max_long, int min_long, int interp, ProcLib lib) { +bool LimitLong::Run(Mat* mat, int max_long, int min_long, int interp, + ProcLib lib) { auto l = LimitLong(max_long, min_long, interp); return l(mat, lib); } diff --git a/fastdeploy/vision/common/processors/limit_long.h b/fastdeploy/vision/common/processors/limit_long.h index e21ddbf2f4..49055973d2 100644 --- a/fastdeploy/vision/common/processors/limit_long.h +++ b/fastdeploy/vision/common/processors/limit_long.h @@ -34,12 +34,12 @@ class FASTDEPLOY_DECL LimitLong : public Processor { // to min_long_, while scale the short edge proportionally. bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "LimitLong"; } static bool Run(Mat* mat, int max_long = -1, int min_long = -1, - int interp = 1, ProcLib lib = ProcLib::OPENCV); + int interp = 1, ProcLib lib = ProcLib::DEFAULT); int GetMaxLong() const { return max_long_; } private: diff --git a/fastdeploy/vision/common/processors/limit_short.cc b/fastdeploy/vision/common/processors/limit_short.cc index 56eb234f8b..d0f0697c8f 100644 --- a/fastdeploy/vision/common/processors/limit_short.cc +++ b/fastdeploy/vision/common/processors/limit_short.cc @@ -41,8 +41,8 @@ bool LimitShort::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool LimitShort::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool LimitShort::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); int origin_w = im->width(); int origin_h = im->height(); int im_size_min = std::min(origin_w, origin_h); @@ -65,7 +65,7 @@ bool LimitShort::ImplByFalconCV(Mat* mat) { } else if (interp_ == 2) { interp_method = fcv::InterpolationType::INTER_CUBIC; } else { - FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but " + FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but " "now it's " << interp_ << "." << std::endl; return false; @@ -81,7 +81,8 @@ bool LimitShort::ImplByFalconCV(Mat* mat) { } #endif -bool LimitShort::Run(Mat* mat, int max_short, int min_short, int interp, ProcLib lib) { +bool LimitShort::Run(Mat* mat, int max_short, int min_short, int interp, + ProcLib lib) { auto l = LimitShort(max_short, min_short, interp); return l(mat, lib); } diff --git a/fastdeploy/vision/common/processors/limit_short.h b/fastdeploy/vision/common/processors/limit_short.h index 8f6af34673..268fa088fe 100644 --- a/fastdeploy/vision/common/processors/limit_short.h +++ b/fastdeploy/vision/common/processors/limit_short.h @@ -34,12 +34,12 @@ class LimitShort : public Processor { // to min_short_, while scale the long edge proportionally. bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "LimitShort"; } static bool Run(Mat* mat, int max_short = -1, int min_short = -1, - int interp = 1, ProcLib lib = ProcLib::OPENCV); + int interp = 1, ProcLib lib = ProcLib::DEFAULT); int GetMaxShort() const { return max_short_; } private: diff --git a/fastdeploy/vision/common/processors/mat.cc b/fastdeploy/vision/common/processors/mat.cc index a906ff760a..db380ba740 100644 --- a/fastdeploy/vision/common/processors/mat.cc +++ b/fastdeploy/vision/common/processors/mat.cc @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "fastdeploy/vision/common/processors/mat.h" -#include "fastdeploy/vision/common/processors/utils.h" + #include "fastdeploy/utils/utils.h" +#include "fastdeploy/vision/common/processors/utils.h" namespace fastdeploy { namespace vision { @@ -23,15 +24,16 @@ void* Mat::Data() { #ifdef ENABLE_FLYCV return fcv_mat.data(); #else - FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat."); + FDASSERT(false, + "FastDeploy didn't compile with FlyCV, but met data type with " + "fcv::Mat."); #endif } return cpu_mat.ptr(); } void Mat::ShareWithTensor(FDTensor* tensor) { - tensor->SetExternalData({Channels(), Height(), Width()}, Type(), - Data()); + tensor->SetExternalData({Channels(), Height(), Width()}, Type(), Data()); tensor->device = Device::CPU; if (layout == Layout::HWC) { tensor->shape = {Height(), Width(), Channels()}; @@ -56,26 +58,28 @@ void Mat::PrintInfo(const std::string& flag) { #ifdef ENABLE_FLYCV fcv::Scalar mean = fcv::mean(fcv_mat); std::cout << flag << ": " - << "DataType=" << Type() << ", " - << "Channel=" << Channels() << ", " - << "Height=" << Height() << ", " - << "Width=" << Width() << ", " - << "Mean="; + << "DataType=" << Type() << ", " + << "Channel=" << Channels() << ", " + << "Height=" << Height() << ", " + << "Width=" << Width() << ", " + << "Mean="; for (int i = 0; i < Channels(); ++i) { std::cout << mean[i] << " "; } std::cout << std::endl; #else - FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat."); + FDASSERT(false, + "FastDeploy didn't compile with FlyCV, but met data type with " + "fcv::Mat."); #endif } else { cv::Scalar mean = cv::mean(cpu_mat); std::cout << flag << ": " - << "DataType=" << Type() << ", " - << "Channel=" << Channels() << ", " - << "Height=" << Height() << ", " - << "Width=" << Width() << ", " - << "Mean="; + << "DataType=" << Type() << ", " + << "Channel=" << Channels() << ", " + << "Height=" << Height() << ", " + << "Width=" << Width() << ", " + << "Mean="; for (int i = 0; i < Channels(); ++i) { std::cout << mean[i] << " "; } @@ -84,87 +88,35 @@ void Mat::PrintInfo(const std::string& flag) { } FDDataType Mat::Type() { - int type = -1; + int type = -1; if (mat_type == ProcLib::FLYCV) { #ifdef ENABLE_FLYCV - return FalconCVDataTypeToFD(fcv_mat.type()); + return FlyCVDataTypeToFD(fcv_mat.type()); #else - FDASSERT(false, "FastDeploy didn't compile with FalconCV, but met data type with fcv::Mat."); + FDASSERT(false, + "FastDeploy didn't compile with FlyCV, but met data type with " + "fcv::Mat."); #endif } return OpenCVDataTypeToFD(cpu_mat.type()); } -Mat CreateFromTensor(const FDTensor& tensor) { - int type = tensor.dtype; - cv::Mat temp_mat; - FDASSERT(tensor.shape.size() == 3, - "When create FD Mat from tensor, tensor shape should be 3-Dim, HWC " - "layout"); - int64_t height = tensor.shape[0]; - int64_t width = tensor.shape[1]; - int64_t channel = tensor.shape[2]; - switch (type) { - case FDDataType::UINT8: - temp_mat = cv::Mat(height, width, CV_8UC(channel), - const_cast(tensor.Data())); - break; - - case FDDataType::INT8: - temp_mat = cv::Mat(height, width, CV_8SC(channel), - const_cast(tensor.Data())); - break; - - case FDDataType::INT16: - temp_mat = cv::Mat(height, width, CV_16SC(channel), - const_cast(tensor.Data())); +std::ostream& operator<<(std::ostream& out, const ProcLib& p) { + switch (p) { + case ProcLib::DEFAULT: + out << "ProcLib::DEFAULT"; break; - - case FDDataType::INT32: - temp_mat = cv::Mat(height, width, CV_32SC(channel), - const_cast(tensor.Data())); + case ProcLib::OPENCV: + out << "ProcLib::OPENCV"; break; - - case FDDataType::FP32: - temp_mat = cv::Mat(height, width, CV_32FC(channel), - const_cast(tensor.Data())); + case ProcLib::FLYCV: + out << "ProcLib::FLYCV"; break; - - case FDDataType::FP64: - temp_mat = cv::Mat(height, width, CV_64FC(channel), - const_cast(tensor.Data())); - break; - default: - FDASSERT( - false, - "Tensor type %d is not supported While calling CreateFromTensor.", - type); - break; - } - Mat mat = Mat(temp_mat); - return mat; -} - -std::ostream& operator<<(std::ostream& out,const ProcLib& p) { - switch (p) { - case ProcLib::DEFAULT: - out << "ProcLib::DEFAULT"; - break; - case ProcLib::OPENCV: - out << "ProcLib::OPENCV"; - break; - case ProcLib::FLYCV: - out << "ProcLib::FLYCV"; - break; - default: - FDASSERT(false, "Unknow type of ProcLib."); + FDASSERT(false, "Unknow type of ProcLib."); } return out; } - - - } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/mat.h b/fastdeploy/vision/common/processors/mat.h index 20e56211db..995d497ccf 100644 --- a/fastdeploy/vision/common/processors/mat.h +++ b/fastdeploy/vision/common/processors/mat.h @@ -13,19 +13,19 @@ // limitations under the License. #pragma once #include "fastdeploy/core/fd_tensor.h" -#include "opencv2/core/core.hpp" #include "fastdeploy/vision/common/processors/utils.h" +#include "opencv2/core/core.hpp" namespace fastdeploy { namespace vision { -enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV}; +enum class FASTDEPLOY_DECL ProcLib { DEFAULT, OPENCV, FLYCV }; enum Layout { HWC, CHW }; FASTDEPLOY_DECL std::ostream& operator<<(std::ostream& out, const ProcLib& p); struct FASTDEPLOY_DECL Mat { - explicit Mat(cv::Mat& mat) { + explicit Mat(const cv::Mat& mat) { cpu_mat = mat; layout = Layout::HWC; height = cpu_mat.rows; @@ -34,7 +34,18 @@ struct FASTDEPLOY_DECL Mat { mat_type = ProcLib::OPENCV; } - // careful if you use this interface +#ifdef ENABLE_FLYCV + explicit Mat(const fcv::Mat& mat) { + fcv_mat = mat; + layout = Layout::HWC; + height = fcv_mat.height(); + width = fcv_mat.width(); + channels = fcv_mat.channels(); + mat_type = ProcLib::FLYCV; + } +#endif + + // Careful if you use this interface // this only used if you don't want to write // the original data, and write to a new cv::Mat // then replace the old cv::Mat of this structure @@ -43,15 +54,23 @@ struct FASTDEPLOY_DECL Mat { mat_type = ProcLib::OPENCV; } - inline cv::Mat* GetOpenCVMat() { - FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure."); - return &cpu_mat; - } - - - inline const cv::Mat* GetOpenCVMat() const { - FDASSERT(mat_type == ProcLib::OPENCV, "Met non cv::Mat data structure."); - return &cpu_mat; + cv::Mat* GetOpenCVMat() { + if (mat_type == ProcLib::OPENCV) { + return &cpu_mat; + } else if (mat_type == ProcLib::FLYCV) { +#ifdef ENABLE_FLYCV + // Just a reference to fcv_mat, zero copy. After you + // call this method, cpu_mat and fcv_mat will point + // to the same memory buffer. + cpu_mat = ConvertFlyCVMatToOpenCV(fcv_mat); + mat_type = ProcLib::OPENCV; + return &cpu_mat; +#else + FDASSERT(false, "FastDeploy didn't compiled with FlyCV!"); +#endif + } else { + FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT"); + } } #ifdef ENABLE_FLYCV @@ -60,9 +79,19 @@ struct FASTDEPLOY_DECL Mat { mat_type = ProcLib::FLYCV; } - inline fcv::Mat* GetFalconCVMat() { - FDASSERT(mat_type == ProcLib::FLYCV, "Met non fcv::Mat data strucure."); - return &fcv_mat; + fcv::Mat* GetFlyCVMat() { + if (mat_type == ProcLib::FLYCV) { + return &fcv_mat; + } else if (mat_type == ProcLib::OPENCV) { + // Just a reference to cpu_mat, zero copy. After you + // call this method, fcv_mat and cpu_mat will point + // to the same memory buffer. + fcv_mat = ConvertOpenCVMatToFlyCV(cpu_mat); + mat_type = ProcLib::FLYCV; + return &fcv_mat; + } else { + FDASSERT(false, "The mat_type of custom Mat can not be ProcLib::DEFAULT"); + } } #endif @@ -73,17 +102,11 @@ struct FASTDEPLOY_DECL Mat { int height; int width; cv::Mat cpu_mat; - #ifdef ENABLE_FLYCV fcv::Mat fcv_mat; #endif public: - template - T* GetMat() { - return &cpu_mat; - } - FDDataType Type(); int Channels() const { return channels; } int Width() const { return width; } @@ -97,18 +120,16 @@ struct FASTDEPLOY_DECL Mat { // Only support copy to cpu tensor now bool CopyToTensor(FDTensor* tensor); - // debug functions - // TODO(jiangjiajun) Develop a right process pipeline with c++ is not a easy - // things - // Will add more debug function here to help debug processed image - // This function will print shape / mean of each channels of the Mat + // Debug functions + // TODO(jiangjiajun) Develop a right process pipeline with c++ + // is not a easy things, Will add more debug function here to + // help debug processed image. This function will print shape + // and mean of each channels of the Mat void PrintInfo(const std::string& flag); ProcLib mat_type = ProcLib::OPENCV; Layout layout = Layout::HWC; }; -Mat CreateFromTensor(const FDTensor& tensor); - } // namespace vision } // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/normalize.cc b/fastdeploy/vision/common/processors/normalize.cc index 88d68a65ee..cf4888613c 100644 --- a/fastdeploy/vision/common/processors/normalize.cc +++ b/fastdeploy/vision/common/processors/normalize.cc @@ -65,27 +65,29 @@ bool Normalize::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool Normalize::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool Normalize::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { - FDERROR << "Only supports 3-channels image in FalconCV, but now it's " << im->channels() << "." << std::endl; + FDERROR << "Only supports 3-channels image in FlyCV, but now it's " + << im->channels() << "." << std::endl; return false; } std::vector mean(3, 0); std::vector std(3, 0); for (size_t i = 0; i < 3; ++i) { - std[i] = 1.0 / alpha_[i]; + std[i] = 1.0 / alpha_[i]; mean[i] = -1 * beta_[i] * std[i]; } - fcv::Mat new_im(im->width(), im->height(), fcv::FCVImageType::PACKAGE_BGR_F32); - fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector(), new_im, true); + fcv::Mat new_im(im->width(), im->height(), + fcv::FCVImageType::PACKAGE_BGR_F32); + fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector(), + new_im, true); mat->SetMat(new_im); return true; } #endif - bool Normalize::Run(Mat* mat, const std::vector& mean, const std::vector& std, bool is_scale, const std::vector& min, @@ -94,5 +96,5 @@ bool Normalize::Run(Mat* mat, const std::vector& mean, return n(mat, lib); } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/normalize.h b/fastdeploy/vision/common/processors/normalize.h index 964c19fb2a..515fcd7e6c 100644 --- a/fastdeploy/vision/common/processors/normalize.h +++ b/fastdeploy/vision/common/processors/normalize.h @@ -26,7 +26,7 @@ class FASTDEPLOY_DECL Normalize : public Processor { const std::vector& max = std::vector()); bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "Normalize"; } @@ -44,14 +44,10 @@ class FASTDEPLOY_DECL Normalize : public Processor { const std::vector& std, bool is_scale = true, const std::vector& min = std::vector(), const std::vector& max = std::vector(), - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); - std::vector GetAlpha() const { - return alpha_; - } - std::vector GetBeta() const { - return beta_; - } + std::vector GetAlpha() const { return alpha_; } + std::vector GetBeta() const { return beta_; } private: std::vector alpha_; diff --git a/fastdeploy/vision/common/processors/normalize_and_permute.cc b/fastdeploy/vision/common/processors/normalize_and_permute.cc index 64daf07837..cb78cc7205 100644 --- a/fastdeploy/vision/common/processors/normalize_and_permute.cc +++ b/fastdeploy/vision/common/processors/normalize_and_permute.cc @@ -17,11 +17,11 @@ namespace fastdeploy { namespace vision { - NormalizeAndPermute::NormalizeAndPermute(const std::vector& mean, - const std::vector& std, bool is_scale, - const std::vector& min, - const std::vector& max) { + const std::vector& std, + bool is_scale, + const std::vector& min, + const std::vector& max) { FDASSERT(mean.size() == std.size(), "Normalize: requires the size of mean equal to the size of std."); std::vector mean_(mean.begin(), mean.end()); @@ -65,7 +65,10 @@ bool NormalizeAndPermute::ImplByOpenCV(Mat* mat) { } cv::Mat res(origin_h, origin_w, CV_32FC(im->channels())); for (int i = 0; i < im->channels(); ++i) { - cv::extractChannel(split_im[i], cv::Mat(origin_h, origin_w, CV_32FC1, res.ptr() + i * origin_h * origin_w * 4), 0); + cv::extractChannel(split_im[i], + cv::Mat(origin_h, origin_w, CV_32FC1, + res.ptr() + i * origin_h * origin_w * 4), + 0); } mat->SetMat(res); @@ -74,38 +77,39 @@ bool NormalizeAndPermute::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool NormalizeAndPermute::ImplByFalconCV(Mat* mat) { +bool NormalizeAndPermute::ImplByFlyCV(Mat* mat) { if (mat->layout != Layout::HWC) { FDERROR << "Only supports input with HWC layout." << std::endl; return false; } - fcv::Mat* im = mat->GetFalconCVMat(); + fcv::Mat* im = mat->GetFlyCVMat(); if (im->channels() != 3) { - FDERROR << "Only supports 3-channels image in FalconCV, but now it's " << im->channels() << "." << std::endl; + FDERROR << "Only supports 3-channels image in FlyCV, but now it's " + << im->channels() << "." << std::endl; return false; } std::vector mean(3, 0); std::vector std(3, 0); for (size_t i = 0; i < 3; ++i) { - std[i] = 1.0 / alpha_[i]; + std[i] = 1.0 / alpha_[i]; mean[i] = -1 * beta_[i] * std[i]; } fcv::Mat new_im; - fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector(), new_im, false); + fcv::normalize_to_submean_to_reorder(*im, mean, std, std::vector(), + new_im, false); mat->SetMat(new_im); mat->layout = Layout::CHW; return true; } #endif - bool NormalizeAndPermute::Run(Mat* mat, const std::vector& mean, - const std::vector& std, bool is_scale, - const std::vector& min, - const std::vector& max, ProcLib lib) { + const std::vector& std, bool is_scale, + const std::vector& min, + const std::vector& max, ProcLib lib) { auto n = NormalizeAndPermute(mean, std, is_scale, min, max); return n(mat, lib); } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/normalize_and_permute.h b/fastdeploy/vision/common/processors/normalize_and_permute.h index 4a154f71f8..ec47665262 100644 --- a/fastdeploy/vision/common/processors/normalize_and_permute.h +++ b/fastdeploy/vision/common/processors/normalize_and_permute.h @@ -21,13 +21,12 @@ namespace vision { class FASTDEPLOY_DECL NormalizeAndPermute : public Processor { public: NormalizeAndPermute(const std::vector& mean, - const std::vector& std, - bool is_scale = true, - const std::vector& min = std::vector(), - const std::vector& max = std::vector()); + const std::vector& std, bool is_scale = true, + const std::vector& min = std::vector(), + const std::vector& max = std::vector()); bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "NormalizeAndPermute"; } @@ -45,7 +44,7 @@ class FASTDEPLOY_DECL NormalizeAndPermute : public Processor { const std::vector& std, bool is_scale = true, const std::vector& min = std::vector(), const std::vector& max = std::vector(), - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); void SetAlpha(const std::vector& alpha) { alpha_.clear(); diff --git a/fastdeploy/vision/common/processors/pad.cc b/fastdeploy/vision/common/processors/pad.cc index 7352356d02..5068b16fc1 100644 --- a/fastdeploy/vision/common/processors/pad.cc +++ b/fastdeploy/vision/common/processors/pad.cc @@ -53,7 +53,7 @@ bool Pad::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool Pad::ImplByFalconCV(Mat* mat) { +bool Pad::ImplByFlyCV(Mat* mat) { if (mat->layout != Layout::HWC) { FDERROR << "Pad: The input data must be Layout::HWC format!" << std::endl; return false; @@ -70,7 +70,7 @@ bool Pad::ImplByFalconCV(Mat* mat) { << std::endl; return false; } - fcv::Mat* im = mat->GetFalconCVMat(); + fcv::Mat* im = mat->GetFlyCVMat(); fcv::Scalar value; if (value_.size() == 1) { value = fcv::Scalar(value_[0]); @@ -83,7 +83,7 @@ bool Pad::ImplByFalconCV(Mat* mat) { } fcv::Mat new_im; fcv::copy_make_border(*im, new_im, top_, bottom_, left_, right_, - fcv::BorderTypes::BORDER_CONSTANT, value); + fcv::BorderTypes::BORDER_CONSTANT, value); mat->SetMat(new_im); mat->SetHeight(new_im.height()); mat->SetWidth(new_im.width()); @@ -92,11 +92,10 @@ bool Pad::ImplByFalconCV(Mat* mat) { #endif bool Pad::Run(Mat* mat, const int& top, const int& bottom, const int& left, - const int& right, const std::vector& value, - ProcLib lib) { + const int& right, const std::vector& value, ProcLib lib) { auto p = Pad(top, bottom, left, right, value); return p(mat, lib); } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/pad.h b/fastdeploy/vision/common/processors/pad.h index 5405564ec4..661632e77c 100644 --- a/fastdeploy/vision/common/processors/pad.h +++ b/fastdeploy/vision/common/processors/pad.h @@ -31,13 +31,13 @@ class FASTDEPLOY_DECL Pad : public Processor { } bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "Pad"; } static bool Run(Mat* mat, const int& top, const int& bottom, const int& left, const int& right, const std::vector& value, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: int top_; diff --git a/fastdeploy/vision/common/processors/pad_to_size.cc b/fastdeploy/vision/common/processors/pad_to_size.cc index 5956953e04..3480345091 100644 --- a/fastdeploy/vision/common/processors/pad_to_size.cc +++ b/fastdeploy/vision/common/processors/pad_to_size.cc @@ -76,7 +76,7 @@ bool PadToSize::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool PadToSize::ImplByFalconCV(Mat* mat) { +bool PadToSize::ImplByFlyCV(Mat* mat) { if (width_ == -1 || height_ == -1) { return true; } @@ -115,7 +115,7 @@ bool PadToSize::ImplByFalconCV(Mat* mat) { return true; } - fcv::Mat* im = mat->GetFalconCVMat(); + fcv::Mat* im = mat->GetFlyCVMat(); fcv::Scalar value; if (value_.size() == 1) { value = fcv::Scalar(value_[0]); @@ -128,8 +128,9 @@ bool PadToSize::ImplByFalconCV(Mat* mat) { } fcv::Mat new_im; // top, bottom, left, right - fcv::copy_make_border(*im, new_im, 0, height_ - origin_h, 0, width_ - origin_w, - fcv::BorderTypes::BORDER_CONSTANT, value); + fcv::copy_make_border(*im, new_im, 0, height_ - origin_h, 0, + width_ - origin_w, fcv::BorderTypes::BORDER_CONSTANT, + value); mat->SetMat(new_im); mat->SetHeight(height_); mat->SetWidth(width_); diff --git a/fastdeploy/vision/common/processors/pad_to_size.h b/fastdeploy/vision/common/processors/pad_to_size.h index dff8110891..c73cee3c24 100644 --- a/fastdeploy/vision/common/processors/pad_to_size.h +++ b/fastdeploy/vision/common/processors/pad_to_size.h @@ -29,13 +29,13 @@ class FASTDEPLOY_DECL PadToSize : public Processor { } bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "PadToSize"; } static bool Run(Mat* mat, int width, int height, const std::vector& value, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: int width_; diff --git a/fastdeploy/vision/common/processors/resize.cc b/fastdeploy/vision/common/processors/resize.cc index eb1932abc0..28488c2cd4 100644 --- a/fastdeploy/vision/common/processors/resize.cc +++ b/fastdeploy/vision/common/processors/resize.cc @@ -55,12 +55,12 @@ bool Resize::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool Resize::ImplByFalconCV(Mat* mat) { +bool Resize::ImplByFlyCV(Mat* mat) { if (mat->layout != Layout::HWC) { FDERROR << "Resize: The format of input is not HWC." << std::endl; return false; } - fcv::Mat* im = mat->GetFalconCVMat(); + fcv::Mat* im = mat->GetFlyCVMat(); int origin_w = im->width(); int origin_h = im->height(); @@ -79,7 +79,7 @@ bool Resize::ImplByFalconCV(Mat* mat) { } else if (interp_ == 2) { interp_method = fcv::InterpolationType::INTER_CUBIC; } else { - FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but " + FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but " "now it's " << interp_ << "." << std::endl; return false; @@ -99,7 +99,8 @@ bool Resize::ImplByFalconCV(Mat* mat) { mat->SetHeight(new_im.height()); } else if (scale_w_ > 0 && scale_h_ > 0) { fcv::Mat new_im; - fcv::resize(*im, new_im, fcv::Size(0, 0), scale_w_, scale_h_, interp_method); + fcv::resize(*im, new_im, fcv::Size(0, 0), scale_w_, scale_h_, + interp_method); mat->SetMat(new_im); mat->SetWidth(new_im.width()); mat->SetHeight(new_im.height()); @@ -122,5 +123,5 @@ bool Resize::Run(Mat* mat, int width, int height, float scale_w, float scale_h, return r(mat, lib); } -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/resize.h b/fastdeploy/vision/common/processors/resize.h index b20f0e468c..e6a4ba1b05 100644 --- a/fastdeploy/vision/common/processors/resize.h +++ b/fastdeploy/vision/common/processors/resize.h @@ -33,13 +33,13 @@ class FASTDEPLOY_DECL Resize : public Processor { bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "Resize"; } static bool Run(Mat* mat, int width, int height, float scale_w = -1.0, float scale_h = -1.0, int interp = 1, bool use_scale = false, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); bool SetWidthAndHeight(int width, int height) { width_ = width; diff --git a/fastdeploy/vision/common/processors/resize_by_long.h b/fastdeploy/vision/common/processors/resize_by_long.h index 033bc9d5ba..c288e07a43 100644 --- a/fastdeploy/vision/common/processors/resize_by_long.h +++ b/fastdeploy/vision/common/processors/resize_by_long.h @@ -33,7 +33,7 @@ class ResizeByLong : public Processor { static bool Run(Mat* mat, int target_size, int interp = 1, bool use_scale = true, int max_size = -1, - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: double GenerateScale(const int origin_w, const int origin_h); diff --git a/fastdeploy/vision/common/processors/resize_by_short.cc b/fastdeploy/vision/common/processors/resize_by_short.cc index 2fe60b9ee8..5616961f1c 100644 --- a/fastdeploy/vision/common/processors/resize_by_short.cc +++ b/fastdeploy/vision/common/processors/resize_by_short.cc @@ -37,8 +37,8 @@ bool ResizeByShort::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool ResizeByShort::ImplByFalconCV(Mat* mat) { - fcv::Mat* im = mat->GetFalconCVMat(); +bool ResizeByShort::ImplByFlyCV(Mat* mat) { + fcv::Mat* im = mat->GetFlyCVMat(); int origin_w = im->width(); int origin_h = im->height(); double scale = GenerateScale(origin_w, origin_h); @@ -51,7 +51,7 @@ bool ResizeByShort::ImplByFalconCV(Mat* mat) { } else if (interp_ == 2) { interp_method = fcv::InterpolationType::INTER_CUBIC; } else { - FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FalconCV, but " + FDERROR << "LimitLong: Only support interp_ be 0/1/2 with FlyCV, but " "now it's " << interp_ << "." << std::endl; return false; diff --git a/fastdeploy/vision/common/processors/resize_by_short.h b/fastdeploy/vision/common/processors/resize_by_short.h index 2ea901ee39..151605bebd 100644 --- a/fastdeploy/vision/common/processors/resize_by_short.h +++ b/fastdeploy/vision/common/processors/resize_by_short.h @@ -30,14 +30,14 @@ class FASTDEPLOY_DECL ResizeByShort : public Processor { } bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "ResizeByShort"; } static bool Run(Mat* mat, int target_size, int interp = 1, bool use_scale = true, const std::vector& max_hw = std::vector(), - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: double GenerateScale(const int origin_w, const int origin_h); diff --git a/fastdeploy/vision/common/processors/stride_pad.cc b/fastdeploy/vision/common/processors/stride_pad.cc index 4b71e34fbe..062b4f22c1 100644 --- a/fastdeploy/vision/common/processors/stride_pad.cc +++ b/fastdeploy/vision/common/processors/stride_pad.cc @@ -64,7 +64,7 @@ bool StridePad::ImplByOpenCV(Mat* mat) { } #ifdef ENABLE_FLYCV -bool StridePad::ImplByFalconCV(Mat* mat) { +bool StridePad::ImplByFlyCV(Mat* mat) { if (mat->layout != Layout::HWC) { FDERROR << "StridePad: The input data must be Layout::HWC format!" << std::endl; @@ -92,7 +92,7 @@ bool StridePad::ImplByFalconCV(Mat* mat) { if (pad_h == 0 && pad_w == 0) { return true; } - fcv::Mat* im = mat->GetFalconCVMat(); + fcv::Mat* im = mat->GetFlyCVMat(); fcv::Scalar value; if (value_.size() == 1) { value = fcv::Scalar(value_[0]); @@ -105,7 +105,8 @@ bool StridePad::ImplByFalconCV(Mat* mat) { } fcv::Mat new_im; // top, bottom, left, right - fcv::copy_make_border(*im, new_im, 0, pad_h, 0, pad_w, fcv::BorderTypes::BORDER_CONSTANT, value); + fcv::copy_make_border(*im, new_im, 0, pad_h, 0, pad_w, + fcv::BorderTypes::BORDER_CONSTANT, value); mat->SetMat(new_im); mat->SetHeight(new_im.height()); mat->SetWidth(new_im.width()); diff --git a/fastdeploy/vision/common/processors/stride_pad.h b/fastdeploy/vision/common/processors/stride_pad.h index 67fe692d48..18eebd54e1 100644 --- a/fastdeploy/vision/common/processors/stride_pad.h +++ b/fastdeploy/vision/common/processors/stride_pad.h @@ -28,13 +28,13 @@ class FASTDEPLOY_DECL StridePad : public Processor { } bool ImplByOpenCV(Mat* mat); #ifdef ENABLE_FLYCV - bool ImplByFalconCV(Mat* mat); + bool ImplByFlyCV(Mat* mat); #endif std::string Name() { return "StridePad"; } static bool Run(Mat* mat, int stride, const std::vector& value = std::vector(), - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: int stride_ = 32; diff --git a/fastdeploy/vision/common/processors/transform.h b/fastdeploy/vision/common/processors/transform.h index 8d1b5010d0..7952caca34 100644 --- a/fastdeploy/vision/common/processors/transform.h +++ b/fastdeploy/vision/common/processors/transform.h @@ -20,59 +20,64 @@ #include "fastdeploy/vision/common/processors/convert.h" #include "fastdeploy/vision/common/processors/crop.h" #include "fastdeploy/vision/common/processors/hwc2chw.h" +#include "fastdeploy/vision/common/processors/letter_box.h" #include "fastdeploy/vision/common/processors/limit_by_stride.h" #include "fastdeploy/vision/common/processors/limit_long.h" #include "fastdeploy/vision/common/processors/limit_short.h" #include "fastdeploy/vision/common/processors/normalize.h" +#include "fastdeploy/vision/common/processors/normalize_and_permute.h" #include "fastdeploy/vision/common/processors/pad.h" #include "fastdeploy/vision/common/processors/pad_to_size.h" #include "fastdeploy/vision/common/processors/resize.h" #include "fastdeploy/vision/common/processors/resize_by_long.h" #include "fastdeploy/vision/common/processors/resize_by_short.h" #include "fastdeploy/vision/common/processors/stride_pad.h" -#include "fastdeploy/vision/common/processors/normalize_and_permute.h" #include "fastdeploy/vision/common/processors/warp_affine.h" -#include "fastdeploy/vision/common/processors/letter_box.h" namespace fastdeploy { namespace vision { -inline void FuseNormalizeCast(std::vector>* processors) { +inline void FuseNormalizeCast( + std::vector>* processors) { // Fuse Normalize and Cast int cast_index = -1; for (size_t i = 0; i < processors->size(); ++i) { if ((*processors)[i]->Name() == "Cast") { - if (i == 0) { - continue; - } - if ((*processors)[i - 1]->Name() != "Normalize" && (*processors)[i - 1]->Name() != "NormalizeAndPermute") { - continue; - } - cast_index = i; + if (i == 0) { + continue; + } + if ((*processors)[i - 1]->Name() != "Normalize" && + (*processors)[i - 1]->Name() != "NormalizeAndPermute") { + continue; + } + cast_index = i; } } if (cast_index < 0) { return; } - std::cout << dynamic_cast((*processors)[cast_index].get())->GetDtype() << "-----" << std::endl; - if (dynamic_cast((*processors)[cast_index].get())->GetDtype() != "float") { + if (dynamic_cast((*processors)[cast_index].get())->GetDtype() != + "float") { return; } processors->erase(processors->begin() + cast_index); - FDINFO << (*processors)[cast_index - 1]->Name() << " and Cast are fused to " << (*processors)[cast_index - 1]->Name() << " in preprocessing pipeline." << std::endl; + FDINFO << (*processors)[cast_index - 1]->Name() << " and Cast are fused to " + << (*processors)[cast_index - 1]->Name() + << " in preprocessing pipeline." << std::endl; } -inline void FuseNormalizeHWC2CHW(std::vector>* processors) { +inline void FuseNormalizeHWC2CHW( + std::vector>* processors) { // Fuse Normalize and HWC2CHW to NormalizeAndPermute int hwc2chw_index = -1; for (size_t i = 0; i < processors->size(); ++i) { if ((*processors)[i]->Name() == "HWC2CHW") { if (i == 0) { - continue; + continue; } if ((*processors)[i - 1]->Name() != "Normalize") { - continue; + continue; } hwc2chw_index = i; } @@ -83,10 +88,12 @@ inline void FuseNormalizeHWC2CHW(std::vector>* proces } // Get alpha and beta of Normalize - std::vector alpha = dynamic_cast( - (*processors)[hwc2chw_index - 1].get())->GetAlpha(); - std::vector beta = dynamic_cast( - (*processors)[hwc2chw_index - 1].get())->GetBeta(); + std::vector alpha = + dynamic_cast((*processors)[hwc2chw_index - 1].get()) + ->GetAlpha(); + std::vector beta = + dynamic_cast((*processors)[hwc2chw_index - 1].get()) + ->GetBeta(); // Delete Normalize and HWC2CHW processors->erase(processors->begin() + hwc2chw_index); @@ -99,12 +106,13 @@ inline void FuseNormalizeHWC2CHW(std::vector>* proces // Set alpha and beta auto processor = dynamic_cast( - (*processors)[hwc2chw_index - 1].get()); + (*processors)[hwc2chw_index - 1].get()); processor->SetAlpha(alpha); processor->SetBeta(beta); FDINFO << "Normalize and HWC2CHW are fused to NormalizeAndPermute " - " in preprocessing pipeline." << std::endl; + " in preprocessing pipeline." + << std::endl; } inline void FuseTransforms( diff --git a/fastdeploy/vision/common/processors/utils.cc b/fastdeploy/vision/common/processors/utils.cc index 4e5373ef20..ec3eba4727 100644 --- a/fastdeploy/vision/common/processors/utils.cc +++ b/fastdeploy/vision/common/processors/utils.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/utils/utils.h" + #include "fastdeploy/vision/common/processors/utils.h" namespace fastdeploy { @@ -37,12 +38,15 @@ FDDataType OpenCVDataTypeToFD(int type) { } else if (type == 6) { return FDDataType::FP64; } else { - FDASSERT(false, "While calling OpenCVDataTypeToFD(), get type = %d, which is not expected.", type); + FDASSERT(false, + "While calling OpenCVDataTypeToFD(), get type = %d, which is not " + "expected.", + type); } } #ifdef ENABLE_FLYCV -FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type) { +FDDataType FlyCVDataTypeToFD(fcv::FCVImageType type) { if (type == fcv::FCVImageType::GRAY_U8) { return FDDataType::UINT8; } else if (type == fcv::FCVImageType::PACKAGE_BGR_U8) { @@ -104,13 +108,14 @@ FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type) { } else if (type == fcv::FCVImageType::GRAY_F64) { return FDDataType::FP64; } - FDASSERT(false, "While calling FalconDataTypeToFD(), get unexpected type:%d.", int(type)); + FDASSERT(false, "While calling FlyCVDataTypeToFD(), get unexpected type:%d.", + int(type)); return FDDataType::UNKNOWN1; } -fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel) { +fcv::FCVImageType CreateFlyCVDataType(FDDataType type, int channel) { FDASSERT(channel == 1 || channel == 3 || channel == 4, - "Only support channel be 1/3/4 in Falcon."); + "Only support channel be 1/3/4 in FlyCV."); if (type == FDDataType::UINT8) { if (channel == 1) { return fcv::FCVImageType::GRAY_U8; @@ -132,18 +137,54 @@ fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel) { return fcv::FCVImageType::PACKAGE_BGR_F32; } -fcv::Mat ConvertOpenCVMatToFalconCV(cv::Mat& im) { +int CreateOpenCVDataType(FDDataType type, int channel) { + FDASSERT(channel == 1 || channel == 3 || channel == 4, + "Only support channel be 1/3/4 in OpenCV."); + if (type == FDDataType::UINT8) { + if (channel == 1) { + return CV_8UC1; + } else if (channel == 3) { + return CV_8UC3; + } else { + return CV_8UC4; + } + } else if (type == FDDataType::FP32) { + if (channel == 1) { + return CV_32FC1; + } else if (channel == 3) { + return CV_32FC3; + } else { + return CV_32FC4; + } + } + FDASSERT(false, "Data type of %s is not supported.", Str(type).c_str()); + return CV_32FC3; +} + +fcv::Mat ConvertOpenCVMatToFlyCV(cv::Mat& im) { int type = im.type() % 8; // 0: uint8; 5: float32; 6: float64 if (type != 0 && type != 5 && type != 6) { - FDASSERT(false, "Only support type of uint8/float/double, but now it's %d.", im.type()); + FDASSERT(false, "Only support type of uint8/float/double, but now it's %d.", + im.type()); } - auto fcv_type = CreateFalconCVDataType(OpenCVDataTypeToFD(im.type()), im.channels()); - return fcv::Mat(im.cols, im.rows, fcv_type, im.ptr()); + auto fcv_type = + CreateFlyCVDataType(OpenCVDataTypeToFD(im.type()), im.channels()); + return fcv::Mat(im.cols, im.rows, fcv_type, im.ptr()); // reference only } -#endif - +cv::Mat ConvertFlyCVMatToOpenCV(fcv::Mat& fim) { + auto fd_dtype = FlyCVDataTypeToFD(fim.type()); + if (fd_dtype != FDDataType::UINT8 && fd_dtype != FDDataType::FP32 && + fd_dtype != FDDataType::FP64) { + FDASSERT(false, "Only support type of uint8/float/double, but now it's %s.", + Str(fd_dtype).c_str()); + } + auto ocv_type = CreateOpenCVDataType(fd_dtype, fim.channels()); + return cv::Mat(fim.height(), fim.width(), ocv_type, + fim.data()); // reference only +} +#endif -} // namespace vision -} // namespace fastdeploy +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/common/processors/utils.h b/fastdeploy/vision/common/processors/utils.h index 57c1309e7f..3b3cfc40dc 100644 --- a/fastdeploy/vision/common/processors/utils.h +++ b/fastdeploy/vision/common/processors/utils.h @@ -14,12 +14,12 @@ #pragma once +#include "fastdeploy/core/fd_tensor.h" #include "fastdeploy/utils/utils.h" #include "opencv2/core/core.hpp" -#include "fastdeploy/core/fd_tensor.h" #ifdef ENABLE_FLYCV -#include "flycv.h" // NOLINT +#include "flycv.h" // NOLINT #endif namespace fastdeploy { @@ -27,14 +27,18 @@ namespace vision { // Convert data type of opencv to FDDataType FDDataType OpenCVDataTypeToFD(int type); +// Create data type of opencv by FDDataType +int CreateOpenCVDataType(FDDataType type, int channel = 1); #ifdef ENABLE_FLYCV // Convert data type of flycv to FDDataType -FDDataType FalconCVDataTypeToFD(fcv::FCVImageType type); +FDDataType FlyCVDataTypeToFD(fcv::FCVImageType type); // Create data type of flycv by FDDataType -fcv::FCVImageType CreateFalconCVDataType(FDDataType type, int channel = 1); +fcv::FCVImageType CreateFlyCVDataType(FDDataType type, int channel = 1); // Convert cv::Mat to fcv::Mat -fcv::Mat ConvertOpenCVMatToFalconCV(cv::Mat& im); +fcv::Mat ConvertOpenCVMatToFlyCV(cv::Mat& im); +// Convert fcv::Mat to fcv::mat +cv::Mat ConvertFlyCVMatToOpenCV(fcv::Mat& fim); #endif } // namespace vision diff --git a/fastdeploy/vision/common/processors/warp_affine.h b/fastdeploy/vision/common/processors/warp_affine.h index 5548d495fc..4349ee206b 100644 --- a/fastdeploy/vision/common/processors/warp_affine.h +++ b/fastdeploy/vision/common/processors/warp_affine.h @@ -21,9 +21,7 @@ namespace vision { class WarpAffine : public Processor { public: - WarpAffine(const cv::Mat& trans_matrix, - int width, int height, - int interp = 1, + WarpAffine(const cv::Mat& trans_matrix, int width, int height, int interp = 1, int border_mode = 0, const cv::Scalar& borderValue = cv::Scalar()) { trans_matrix_ = trans_matrix; @@ -37,7 +35,7 @@ class WarpAffine : public Processor { bool ImplByOpenCV(Mat* mat); std::string Name() { return "WarpAffine"; } - bool SetTransformMatrix(const cv::Mat &trans_matrix) { + bool SetTransformMatrix(const cv::Mat& trans_matrix) { trans_matrix_ = trans_matrix; return true; } @@ -46,13 +44,10 @@ class WarpAffine : public Processor { return std::make_tuple(width_, height_); } - static bool Run(Mat* mat, - const cv::Mat& trans_matrix, - int width, int height, - int interp = 1, - int border_mode = 0, + static bool Run(Mat* mat, const cv::Mat& trans_matrix, int width, int height, + int interp = 1, int border_mode = 0, const cv::Scalar& borderValue = cv::Scalar(), - ProcLib lib = ProcLib::OPENCV); + ProcLib lib = ProcLib::DEFAULT); private: cv::Mat trans_matrix_; diff --git a/fastdeploy/vision/detection/contrib/yolov5.cc b/fastdeploy/vision/detection/contrib/yolov5.cc index 6ab4646e08..27f74fd551 100644 --- a/fastdeploy/vision/detection/contrib/yolov5.cc +++ b/fastdeploy/vision/detection/contrib/yolov5.cc @@ -66,7 +66,7 @@ YOLOv5::YOLOv5(const std::string& model_file, const std::string& params_file, valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } runtime_option = custom_option; @@ -184,11 +184,14 @@ void YOLOv5::UseCudaPreprocessing(int max_image_size) { use_cuda_preprocessing_ = true; is_scale_up_ = true; if (input_img_cuda_buffer_host_ == nullptr) { - // prepare input data cache in GPU pinned memory - CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, max_image_size * 3)); + // prepare input data cache in GPU pinned memory + CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, + max_image_size * 3)); // prepare input data cache in GPU device memory - CUDA_CHECK(cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); - CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, 3 * size_[0] * size_[1] * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); + CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, + 3 * size_[0] * size_[1] * sizeof(float))); } #else FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON." @@ -197,15 +200,17 @@ void YOLOv5::UseCudaPreprocessing(int max_image_size) { #endif } -bool YOLOv5::CudaPreprocess(Mat* mat, FDTensor* output, - std::map>* im_info, - const std::vector& size, - const std::vector padding_value, - bool is_mini_pad, bool is_no_pad, bool is_scale_up, - int stride, float max_wh, bool multi_label) { +bool YOLOv5::CudaPreprocess( + Mat* mat, FDTensor* output, + std::map>* im_info, + const std::vector& size, const std::vector padding_value, + bool is_mini_pad, bool is_no_pad, bool is_scale_up, int stride, + float max_wh, bool multi_label) { #ifdef ENABLE_CUDA_PREPROCESS if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) { - FDERROR << "Preprocessing with CUDA is only available when the arguments satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." << std::endl; + FDERROR << "Preprocessing with CUDA is only available when the arguments " + "satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." + << std::endl; return false; } @@ -219,14 +224,15 @@ bool YOLOv5::CudaPreprocess(Mat* mat, FDTensor* output, int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels(); memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size); CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_, - input_img_cuda_buffer_host_, - src_img_buf_size, cudaMemcpyHostToDevice, stream)); + input_img_cuda_buffer_host_, src_img_buf_size, + cudaMemcpyHostToDevice, stream)); utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(), mat->Height(), input_tensor_cuda_buffer_device_, size[0], size[1], padding_value, stream); // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(size[0]), static_cast(size[1])}; + (*im_info)["output_shape"] = {static_cast(size[0]), + static_cast(size[1])}; output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32, input_tensor_cuda_buffer_device_); @@ -339,22 +345,21 @@ bool YOLOv5::Postprocess( bool YOLOv5::Predict(cv::Mat* im, DetectionResult* result, float conf_threshold, float nms_iou_threshold) { - Mat mat(*im); std::map> im_info; if (use_cuda_preprocessing_) { - if (!CudaPreprocess(&mat, &reused_input_tensors[0], &im_info, size_, padding_value_, - is_mini_pad_, is_no_pad_, is_scale_up_, stride_, max_wh_, - multi_label_)) { + if (!CudaPreprocess(&mat, &reused_input_tensors[0], &im_info, size_, + padding_value_, is_mini_pad_, is_no_pad_, is_scale_up_, + stride_, max_wh_, multi_label_)) { FDERROR << "Failed to preprocess input image." << std::endl; return false; } } else { - if (!Preprocess(&mat, &reused_input_tensors[0], &im_info, size_, padding_value_, - is_mini_pad_, is_no_pad_, is_scale_up_, stride_, max_wh_, - multi_label_)) { + if (!Preprocess(&mat, &reused_input_tensors[0], &im_info, size_, + padding_value_, is_mini_pad_, is_no_pad_, is_scale_up_, + stride_, max_wh_, multi_label_)) { FDERROR << "Failed to preprocess input image." << std::endl; return false; } diff --git a/fastdeploy/vision/detection/contrib/yolov6.cc b/fastdeploy/vision/detection/contrib/yolov6.cc index 396dab2540..70f79b9f47 100644 --- a/fastdeploy/vision/detection/contrib/yolov6.cc +++ b/fastdeploy/vision/detection/contrib/yolov6.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/detection/contrib/yolov6.h" + #include "fastdeploy/utils/perf.h" #include "fastdeploy/vision/utils/utils.h" #ifdef ENABLE_CUDA_PREPROCESS @@ -69,7 +70,7 @@ YOLOv6::YOLOv6(const std::string& model_file, const std::string& params_file, valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } runtime_option = custom_option; @@ -174,11 +175,14 @@ void YOLOv6::UseCudaPreprocessing(int max_image_size) { use_cuda_preprocessing_ = true; is_scale_up = true; if (input_img_cuda_buffer_host_ == nullptr) { - // prepare input data cache in GPU pinned memory - CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, max_image_size * 3)); + // prepare input data cache in GPU pinned memory + CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, + max_image_size * 3)); // prepare input data cache in GPU device memory - CUDA_CHECK(cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); - CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, 3 * size[0] * size[1] * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); + CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, + 3 * size[0] * size[1] * sizeof(float))); } #else FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON." @@ -187,11 +191,14 @@ void YOLOv6::UseCudaPreprocessing(int max_image_size) { #endif } -bool YOLOv6::CudaPreprocess(Mat* mat, FDTensor* output, - std::map>* im_info) { +bool YOLOv6::CudaPreprocess( + Mat* mat, FDTensor* output, + std::map>* im_info) { #ifdef ENABLE_CUDA_PREPROCESS if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) { - FDERROR << "Preprocessing with CUDA is only available when the arguments satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." << std::endl; + FDERROR << "Preprocessing with CUDA is only available when the arguments " + "satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." + << std::endl; return false; } @@ -205,14 +212,15 @@ bool YOLOv6::CudaPreprocess(Mat* mat, FDTensor* output, int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels(); memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size); CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_, - input_img_cuda_buffer_host_, - src_img_buf_size, cudaMemcpyHostToDevice, stream)); + input_img_cuda_buffer_host_, src_img_buf_size, + cudaMemcpyHostToDevice, stream)); utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(), mat->Height(), input_tensor_cuda_buffer_device_, size[0], size[1], padding_value, stream); // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(size[0]), static_cast(size[1])}; + (*im_info)["output_shape"] = {static_cast(size[0]), + static_cast(size[1])}; output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32, input_tensor_cuda_buffer_device_); diff --git a/fastdeploy/vision/detection/contrib/yolov7.cc b/fastdeploy/vision/detection/contrib/yolov7.cc index 295ee688e3..c3fc1de414 100644 --- a/fastdeploy/vision/detection/contrib/yolov7.cc +++ b/fastdeploy/vision/detection/contrib/yolov7.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/detection/contrib/yolov7.h" + #include "fastdeploy/utils/perf.h" #include "fastdeploy/vision/utils/utils.h" #ifdef ENABLE_CUDA_PREPROCESS @@ -67,7 +68,7 @@ YOLOv7::YOLOv7(const std::string& model_file, const std::string& params_file, valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { - valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; + valid_cpu_backends = {Backend::PDINFER, Backend::ORT, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; } runtime_option = custom_option; @@ -172,11 +173,14 @@ void YOLOv7::UseCudaPreprocessing(int max_image_size) { use_cuda_preprocessing_ = true; is_scale_up = true; if (input_img_cuda_buffer_host_ == nullptr) { - // prepare input data cache in GPU pinned memory - CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, max_image_size * 3)); + // prepare input data cache in GPU pinned memory + CUDA_CHECK(cudaMallocHost((void**)&input_img_cuda_buffer_host_, + max_image_size * 3)); // prepare input data cache in GPU device memory - CUDA_CHECK(cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); - CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, 3 * size[0] * size[1] * sizeof(float))); + CUDA_CHECK( + cudaMalloc((void**)&input_img_cuda_buffer_device_, max_image_size * 3)); + CUDA_CHECK(cudaMalloc((void**)&input_tensor_cuda_buffer_device_, + 3 * size[0] * size[1] * sizeof(float))); } #else FDWARNING << "The FastDeploy didn't compile with BUILD_CUDA_SRC=ON." @@ -185,11 +189,14 @@ void YOLOv7::UseCudaPreprocessing(int max_image_size) { #endif } -bool YOLOv7::CudaPreprocess(Mat* mat, FDTensor* output, - std::map>* im_info) { +bool YOLOv7::CudaPreprocess( + Mat* mat, FDTensor* output, + std::map>* im_info) { #ifdef ENABLE_CUDA_PREPROCESS if (is_mini_pad != false || is_no_pad != false || is_scale_up != true) { - FDERROR << "Preprocessing with CUDA is only available when the arguments satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." << std::endl; + FDERROR << "Preprocessing with CUDA is only available when the arguments " + "satisfy (is_mini_pad=false, is_no_pad=false, is_scale_up=true)." + << std::endl; return false; } @@ -203,14 +210,15 @@ bool YOLOv7::CudaPreprocess(Mat* mat, FDTensor* output, int src_img_buf_size = mat->Height() * mat->Width() * mat->Channels(); memcpy(input_img_cuda_buffer_host_, mat->Data(), src_img_buf_size); CUDA_CHECK(cudaMemcpyAsync(input_img_cuda_buffer_device_, - input_img_cuda_buffer_host_, - src_img_buf_size, cudaMemcpyHostToDevice, stream)); + input_img_cuda_buffer_host_, src_img_buf_size, + cudaMemcpyHostToDevice, stream)); utils::CudaYoloPreprocess(input_img_cuda_buffer_device_, mat->Width(), mat->Height(), input_tensor_cuda_buffer_device_, size[0], size[1], padding_value, stream); // Record output shape of preprocessed image - (*im_info)["output_shape"] = {static_cast(size[0]), static_cast(size[1])}; + (*im_info)["output_shape"] = {static_cast(size[0]), + static_cast(size[1])}; output->SetExternalData({mat->Channels(), size[0], size[1]}, FDDataType::FP32, input_tensor_cuda_buffer_device_); diff --git a/fastdeploy/vision/matting/contrib/modnet.cc b/fastdeploy/vision/matting/contrib/modnet.cc old mode 100755 new mode 100644 index b082665473..b0bc59c66f --- a/fastdeploy/vision/matting/contrib/modnet.cc +++ b/fastdeploy/vision/matting/contrib/modnet.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/matting/contrib/modnet.h" + #include "fastdeploy/utils/perf.h" #include "fastdeploy/vision/utils/utils.h" @@ -26,8 +27,8 @@ MODNet::MODNet(const std::string& model_file, const std::string& params_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { if (model_format == ModelFormat::ONNX) { - valid_cpu_backends = {Backend::ORT}; - valid_gpu_backends = {Backend::ORT, Backend::TRT}; + valid_cpu_backends = {Backend::ORT}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; @@ -118,7 +119,7 @@ bool MODNet::Postprocess( int numel = ipt_h * ipt_w; int nbytes = numel * sizeof(float); result->Resize(numel); - std::memcpy(result->alpha.data(), alpha_resized.GetOpenCVMat()->data, nbytes); + std::memcpy(result->alpha.data(), alpha_resized.Data(), nbytes); return true; } diff --git a/fastdeploy/vision/matting/contrib/rvm.cc b/fastdeploy/vision/matting/contrib/rvm.cc old mode 100755 new mode 100644 index 04b9b93164..ec8ed19fc4 --- a/fastdeploy/vision/matting/contrib/rvm.cc +++ b/fastdeploy/vision/matting/contrib/rvm.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/matting/contrib/rvm.h" + #include "fastdeploy/utils/perf.h" #include "fastdeploy/vision/utils/utils.h" @@ -22,12 +23,13 @@ namespace vision { namespace matting { -RobustVideoMatting::RobustVideoMatting(const std::string& model_file, const std::string& params_file, - const RuntimeOption& custom_option, - const ModelFormat& model_format) { +RobustVideoMatting::RobustVideoMatting(const std::string& model_file, + const std::string& params_file, + const RuntimeOption& custom_option, + const ModelFormat& model_format) { if (model_format == ModelFormat::ONNX) { - valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; - valid_gpu_backends = {Backend::ORT, Backend::TRT}; + valid_cpu_backends = {Backend::OPENVINO, Backend::ORT}; + valid_gpu_backends = {Backend::ORT, Backend::TRT}; } else { valid_cpu_backends = {Backend::PDINFER, Backend::ORT}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; @@ -52,8 +54,9 @@ bool RobustVideoMatting::Initialize() { return true; } -bool RobustVideoMatting::Preprocess(Mat* mat, FDTensor* output, - std::map>* im_info) { +bool RobustVideoMatting::Preprocess( + Mat* mat, FDTensor* output, + std::map>* im_info) { // Resize int resize_w = size[0]; int resize_h = size[1]; @@ -61,7 +64,7 @@ bool RobustVideoMatting::Preprocess(Mat* mat, FDTensor* output, Resize::Run(mat, resize_w, resize_h); } BGR2RGB::Run(mat); - + // Normalize std::vector alpha = {1.0f / 255.0f, 1.0f / 255.0f, 1.0f / 255.0f}; std::vector beta = {0.0f, 0.0f, 0.0f}; @@ -83,7 +86,7 @@ bool RobustVideoMatting::Postprocess( FDASSERT((infer_result.size() == 6), "The default number of output tensor must be 6 according to " "RobustVideoMatting."); - FDTensor& fgr = infer_result.at(0); // fgr (1, 3, h, w) 0.~1. + FDTensor& fgr = infer_result.at(0); // fgr (1, 3, h, w) 0.~1. FDTensor& alpha = infer_result.at(1); // alpha (1, 1, h, w) 0.~1. FDASSERT((fgr.shape[0] == 1), "Only support batch = 1 now."); FDASSERT((alpha.shape[0] == 1), "Only support batch = 1 now."); @@ -98,11 +101,11 @@ bool RobustVideoMatting::Postprocess( // update context if (video_mode) { for (size_t i = 0; i < 4; ++i) { - FDTensor& rki = infer_result.at(i+2); + FDTensor& rki = infer_result.at(i + 2); dynamic_inputs_dims_[i] = rki.shape; dynamic_inputs_datas_[i].resize(rki.Numel()); memcpy(dynamic_inputs_datas_[i].data(), rki.Data(), - rki.Numel() * FDDataTypeSize(rki.dtype)); + rki.Numel() * FDDataTypeSize(rki.dtype)); } } @@ -139,8 +142,8 @@ bool RobustVideoMatting::Postprocess( int numel = in_h * in_w; int nbytes = numel * sizeof(float); result->Resize(numel); - memcpy(result->alpha.data(), alpha_resized.GetOpenCVMat()->data, nbytes); - memcpy(result->foreground.data(), fgr_resized.GetOpenCVMat()->data, nbytes); + memcpy(result->alpha.data(), alpha_resized.Data(), nbytes); + memcpy(result->foreground.data(), fgr_resized.Data(), nbytes); return true; } @@ -154,7 +157,9 @@ bool RobustVideoMatting::Predict(cv::Mat* im, MattingResult* result) { im_info["output_shape"] = {mat.Height(), mat.Width()}; // convert vector to FDTensor for (size_t i = 1; i < inputs_nums; ++i) { - input_tensors[i].SetExternalData(dynamic_inputs_dims_[i-1], FDDataType::FP32, dynamic_inputs_datas_[i-1].data()); + input_tensors[i].SetExternalData(dynamic_inputs_dims_[i - 1], + FDDataType::FP32, + dynamic_inputs_datas_[i - 1].data()); input_tensors[i].device = Device::CPU; } if (!Preprocess(&mat, &input_tensors[0], &im_info)) { diff --git a/fastdeploy/vision/matting/ppmatting/ppmatting.cc b/fastdeploy/vision/matting/ppmatting/ppmatting.cc old mode 100755 new mode 100644 index 5fa63a48dd..cde2183075 --- a/fastdeploy/vision/matting/ppmatting/ppmatting.cc +++ b/fastdeploy/vision/matting/ppmatting/ppmatting.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "fastdeploy/vision/matting/ppmatting/ppmatting.h" + #include "fastdeploy/vision/utils/utils.h" #include "yaml-cpp/yaml.h" @@ -171,15 +172,15 @@ bool PPMatting::Postprocess( std::vector dim{0, 2, 3, 1}; Transpose(alpha_tensor, &alpha_tensor, dim); alpha_tensor.Squeeze(0); - Mat mat = CreateFromTensor(alpha_tensor); + Mat mat = CreateFDMatFromTensor(alpha_tensor); auto iter_ipt = im_info.find("input_shape"); auto iter_out = im_info.find("output_shape"); - if (is_fixed_input_shape_){ + if (is_fixed_input_shape_) { double scale_h = static_cast(iter_out->second[0]) / - static_cast(iter_ipt->second[0]); + static_cast(iter_ipt->second[0]); double scale_w = static_cast(iter_out->second[1]) / - static_cast(iter_ipt->second[1]); + static_cast(iter_ipt->second[1]); double actual_scale = std::min(scale_h, scale_w); int size_before_pad_h = round(actual_scale * iter_ipt->second[0]); @@ -188,7 +189,8 @@ bool PPMatting::Postprocess( Crop::Run(&mat, 0, 0, size_before_pad_w, size_before_pad_h); } - Resize::Run(&mat, iter_ipt->second[1], iter_ipt->second[0]); + Resize::Run(&mat, iter_ipt->second[1], iter_ipt->second[0], -1.0f, -1.0f, 1, + false, ProcLib::OPENCV); result->Clear(); // note: must be setup shape before Resize @@ -197,7 +199,7 @@ bool PPMatting::Postprocess( int numel = iter_ipt->second[0] * iter_ipt->second[1]; int nbytes = numel * sizeof(float); result->Resize(numel); - std::memcpy(result->alpha.data(), mat.GetOpenCVMat()->data, nbytes); + std::memcpy(result->alpha.data(), mat.Data(), nbytes); return true; } diff --git a/fastdeploy/vision/segmentation/ppseg/model.cc b/fastdeploy/vision/segmentation/ppseg/model.cc old mode 100755 new mode 100644 index 3c6318232f..58819ef15c --- a/fastdeploy/vision/segmentation/ppseg/model.cc +++ b/fastdeploy/vision/segmentation/ppseg/model.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/vision/segmentation/ppseg/model.h" + #include "fastdeploy/vision/utils/utils.h" #include "yaml-cpp/yaml.h" @@ -26,7 +27,8 @@ PaddleSegModel::PaddleSegModel(const std::string& model_file, const RuntimeOption& custom_option, const ModelFormat& model_format) { config_file_ = config_file; - valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, Backend::LITE}; + valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::ORT, + Backend::LITE}; valid_gpu_backends = {Backend::PDINFER, Backend::ORT, Backend::TRT}; valid_rknpu_backends = {Backend::RKNPU2}; runtime_option = custom_option; @@ -68,7 +70,7 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() { FDASSERT(op.IsMap(), "Require the transform information in yaml be Map type."); if (op["type"].as() == "Normalize") { - if(!(this->disable_normalize_and_permute)){ + if (!(this->disable_normalize_and_permute)) { std::vector mean = {0.5, 0.5, 0.5}; std::vector std = {0.5, 0.5, 0.5}; if (op["mean"]) { @@ -102,12 +104,13 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() { int input_width = input_shape[3].as(); if (input_height == -1 || input_width == -1) { FDWARNING << "The exported PaddleSeg model is with dynamic shape input, " - << "which is not supported by ONNX Runtime and Tensorrt. " - << "Only OpenVINO and Paddle Inference are available now. " - << "For using ONNX Runtime or Tensorrt, " - << "Please refer to https://github.com/PaddlePaddle/PaddleSeg/blob/develop/docs/model_export.md" - << " to export model with fixed input shape." - << std::endl; + << "which is not supported by ONNX Runtime and Tensorrt. " + << "Only OpenVINO and Paddle Inference are available now. " + << "For using ONNX Runtime or Tensorrt, " + << "Please refer to " + "https://github.com/PaddlePaddle/PaddleSeg/blob/develop/" + "docs/model_export.md" + << " to export model with fixed input shape." << std::endl; valid_cpu_backends = {Backend::OPENVINO, Backend::PDINFER, Backend::LITE}; valid_gpu_backends = {Backend::PDINFER}; } @@ -132,7 +135,7 @@ bool PaddleSegModel::BuildPreprocessPipelineFromConfig() { << "." << std::endl; } } - if(!(this->disable_normalize_and_permute)){ + if (!(this->disable_normalize_and_permute)) { processors_.push_back(std::make_shared()); } return true; @@ -260,7 +263,7 @@ bool PaddleSegModel::Postprocess( infer_result->shape, FDDataType::FP32, static_cast(fp32_result_buffer->data())); } - mat = new Mat(CreateFromTensor(*infer_result)); + mat = new Mat(CreateFDMatFromTensor(*infer_result)); Resize::Run(mat, ipt_w, ipt_h, -1.0f, -1.0f, 1); mat->ShareWithTensor(&new_infer_result); result->shape = new_infer_result.shape; @@ -361,11 +364,13 @@ bool PaddleSegModel::Predict(cv::Mat* im, SegmentationResult* result) { return true; } -void PaddleSegModel::DisableNormalizeAndPermute(){ +void PaddleSegModel::DisableNormalizeAndPermute() { this->disable_normalize_and_permute = true; - // the DisableNormalizeAndPermute function will be invalid if the configuration file is loaded during preprocessing + // the DisableNormalizeAndPermute function will be invalid if the + // configuration file is loaded during preprocessing if (!BuildPreprocessPipelineFromConfig()) { - FDERROR << "Failed to build preprocess pipeline from configuration file." << std::endl; + FDERROR << "Failed to build preprocess pipeline from configuration file." + << std::endl; } } diff --git a/fastdeploy/vision/tracking/pptracking/trajectory.h b/fastdeploy/vision/tracking/pptracking/trajectory.h index 793419ce13..792a5b8a2b 100644 --- a/fastdeploy/vision/tracking/pptracking/trajectory.h +++ b/fastdeploy/vision/tracking/pptracking/trajectory.h @@ -75,7 +75,8 @@ class FASTDEPLOY_DECL Trajectory : public TKalmanFilter { int timestamp, bool update_embedding = true); virtual void activate(int &cnt, int timestamp); - virtual void reactivate(Trajectory *traj, int &cnt, int timestamp, bool newid = false); + virtual void reactivate(Trajectory *traj, int &cnt, + int timestamp, bool newid = false); virtual void mark_lost(void); virtual void mark_removed(void); @@ -229,6 +230,6 @@ inline void Trajectory::mark_lost(void) { state = Lost; } inline void Trajectory::mark_removed(void) { state = Removed; } -} // namespace tracking -} // namespace vision -} // namespace fastdeploy +} // namespace tracking +} // namespace vision +} // namespace fastdeploy diff --git a/fastdeploy/vision/utils/crop_image.cc b/fastdeploy/vision/utils/crop_image.cc index 5d15844b53..1dd545c0f0 100644 --- a/fastdeploy/vision/utils/crop_image.cc +++ b/fastdeploy/vision/utils/crop_image.cc @@ -18,9 +18,9 @@ namespace fastdeploy { namespace vision { namespace utils { -bool CropImageByBox(const Mat& src_im, Mat* dst_im, - const std::vector& box, std::vector* center, - std::vector* scale, const float expandratio) { +bool CropImageByBox(Mat& src_im, Mat* dst_im, + const std::vector& box, std::vector* center, + std::vector* scale, const float expandratio) { const cv::Mat* img = src_im.GetOpenCVMat(); cv::Mat* crop_img = dst_im->GetOpenCVMat(); int xmin = static_cast(box[0]); @@ -34,12 +34,12 @@ bool CropImageByBox(const Mat& src_im, Mat* dst_im, // adjust h or w to keep image ratio, expand the shorter edge if (half_h * 3 > half_w * 4) { half_w = half_h * 0.75; - } - int crop_xmin =std::max(0, static_cast(centerx - half_w)); - int crop_ymin =std::max(0, static_cast(centery - half_h)); + } + int crop_xmin = std::max(0, static_cast(centerx - half_w)); + int crop_ymin = std::max(0, static_cast(centery - half_h)); int crop_xmax = std::min(img->cols - 1, static_cast(centerx + half_w)); int crop_ymax = std::min(img->rows - 1, static_cast(centery + half_h)); - + crop_img->create(crop_ymax - crop_ymin, crop_xmax - crop_xmin, img->type()); *crop_img = (*img)(cv::Range(crop_ymin, crop_ymax), cv::Range(crop_xmin, crop_xmax)); diff --git a/fastdeploy/vision/utils/utils.h b/fastdeploy/vision/utils/utils.h index 4d6a006c32..1590922d8a 100644 --- a/fastdeploy/vision/utils/utils.h +++ b/fastdeploy/vision/utils/utils.h @@ -17,6 +17,7 @@ #include #include #include + #include "fastdeploy/core/fd_tensor.h" #include "fastdeploy/utils/utils.h" #include "fastdeploy/vision/common/result.h" @@ -43,7 +44,7 @@ std::vector TopKIndices(const T* array, int array_size, int topk) { std::vector res(topk); std::set searched; for (int32_t i = 0; i < topk; ++i) { - T min = -99999999; + T min = static_cast(-99999999); for (int32_t j = 0; j < array_size; ++j) { if (searched.find(j) != searched.end()) { continue; @@ -75,23 +76,25 @@ FASTDEPLOY_DECL float CosineSimilarity(const std::vector& a, const std::vector& b, bool normalized = true); -bool CropImageByBox(const Mat& src_im, Mat* dst_im, +bool CropImageByBox(Mat& src_im, Mat* dst_im, const std::vector& box, std::vector* center, std::vector* scale, const float expandratio = 0.3); /** - * Function: for keypoint detection model, fine positioning of keypoints in postprocess - * Parameters: - * heatmap: model inference results for keypoint detection models - * dim: shape information of the inference result - * coords: coordinates after refined positioning - * px: px = int(coords[ch * 2] + 0.5) , refer to API detection::GetFinalPredictions - * py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions - * index: index information of heatmap pixels - * ch: channel - * Paper reference: DARK postpocessing, Zhang et al. Distribution-Aware Coordinate - * Representation for Human Pose Estimation (CVPR 2020). - */ + * Function: for keypoint detection model, fine positioning of keypoints in + * postprocess + * Parameters: + * heatmap: model inference results for keypoint detection models + * dim: shape information of the inference result + * coords: coordinates after refined positioning + * px: px = int(coords[ch * 2] + 0.5) , refer to API detection::GetFinalPredictions + * py: px = int(coords[ch * 2 + 1] + 0.5), refer to API detection::GetFinalPredictions + * index: index information of heatmap pixels + * ch: channel + * Paper reference: DARK postpocessing, Zhang et al. + * Distribution-Aware Coordinate Representation for Human Pose Estimation (CVPR + * 2020). + */ void DarkParse(const std::vector& heatmap, const std::vector& dim, std::vector* coords, const int px, const int py, const int index, const int ch);