From 0805ead0edb530bb9fb0d10c089cd340297f934f Mon Sep 17 00:00:00 2001 From: heliqi <1101791222@qq.com> Date: Thu, 15 Sep 2022 00:09:31 -0500 Subject: [PATCH] add 'GetOutputInfos' and 'GetInputInfos' interface (#232) add GetOutputInfos GetInputInfos --- fastdeploy/backends/backend.h | 7 +++- fastdeploy/backends/openvino/ov_backend.cc | 42 +++++++++++++++----- fastdeploy/backends/openvino/ov_backend.h | 2 + fastdeploy/backends/ort/ort_backend.cc | 18 +++++++++ fastdeploy/backends/ort/ort_backend.h | 2 + fastdeploy/backends/paddle/paddle_backend.cc | 6 ++- fastdeploy/backends/paddle/paddle_backend.h | 4 +- fastdeploy/backends/tensorrt/trt_backend.cc | 17 ++++++++ fastdeploy/backends/tensorrt/trt_backend.h | 2 + fastdeploy/core/fd_tensor.cc | 28 +++++++++++++ fastdeploy/core/fd_tensor.h | 30 ++------------ fastdeploy/fastdeploy_runtime.cc | 9 +++++ fastdeploy/fastdeploy_runtime.h | 2 + 13 files changed, 126 insertions(+), 43 deletions(-) diff --git a/fastdeploy/backends/backend.h b/fastdeploy/backends/backend.h index 6d2e11f662..9abd18e75e 100644 --- a/fastdeploy/backends/backend.h +++ b/fastdeploy/backends/backend.h @@ -14,13 +14,14 @@ #pragma once -#include "fastdeploy/backends/common/multiclass_nms.h" -#include "fastdeploy/core/fd_tensor.h" #include #include #include #include +#include "fastdeploy/backends/common/multiclass_nms.h" +#include "fastdeploy/core/fd_tensor.h" + namespace fastdeploy { struct TensorInfo { @@ -56,6 +57,8 @@ class BaseBackend { virtual int NumOutputs() const = 0; virtual TensorInfo GetInputInfo(int index) = 0; virtual TensorInfo GetOutputInfo(int index) = 0; + virtual std::vector GetInputInfos() = 0; + virtual std::vector GetOutputInfos() = 0; virtual bool Infer(std::vector& inputs, std::vector* outputs) = 0; }; diff --git a/fastdeploy/backends/openvino/ov_backend.cc b/fastdeploy/backends/openvino/ov_backend.cc index 6f8f302bb6..66b08fc6cf 100644 --- a/fastdeploy/backends/openvino/ov_backend.cc +++ b/fastdeploy/backends/openvino/ov_backend.cc @@ -107,19 +107,25 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file, // So here will reorder it's inputs and outputs std::string model_content; ReadBinaryFromFile(model_file, &model_content); - auto reader = paddle2onnx::PaddleReader(model_content.c_str(), model_content.size()); + auto reader = + paddle2onnx::PaddleReader(model_content.c_str(), model_content.size()); if (reader.num_inputs != input_infos.size()) { - FDERROR << "The number of inputs from PaddleReader:" << reader.num_inputs << " not equal to the number of inputs from OpenVINO:" << input_infos.size() << "." << std::endl; + FDERROR << "The number of inputs from PaddleReader:" << reader.num_inputs + << " not equal to the number of inputs from OpenVINO:" + << input_infos.size() << "." << std::endl; return false; } if (reader.num_outputs != output_infos.size()) { - FDERROR << "The number of outputs from PaddleReader:" << reader.num_outputs << " not equal to the number of outputs from OpenVINO:" << output_infos.size() << "." << std::endl; + FDERROR << "The number of outputs from PaddleReader:" << reader.num_outputs + << " not equal to the number of outputs from OpenVINO:" + << output_infos.size() << "." << std::endl; return false; } for (int i = 0; i < reader.num_inputs; ++i) { auto iter = input_infos.find(std::string(reader.inputs[i].name)); if (iter == input_infos.end()) { - FDERROR << "Cannot find input name:" << reader.inputs[i].name << " from OpenVINO model." << std::endl; + FDERROR << "Cannot find input name:" << reader.inputs[i].name + << " from OpenVINO model." << std::endl; return false; } input_infos_.push_back(iter->second); @@ -127,7 +133,8 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file, for (int i = 0; i < reader.num_outputs; ++i) { auto iter = output_infos.find(std::string(reader.outputs[i].name)); if (iter == output_infos.end()) { - FDERROR << "Cannot find output name:" << reader.outputs[i].name << " from OpenVINO model." << std::endl; + FDERROR << "Cannot find output name:" << reader.outputs[i].name + << " from OpenVINO model." << std::endl; return false; } output_infos_.push_back(iter->second); @@ -146,6 +153,14 @@ TensorInfo OpenVINOBackend::GetInputInfo(int index) { return input_infos_[index]; } +std::vector OpenVINOBackend::GetInputInfos() { + return input_infos_; +} + +std::vector OpenVINOBackend::GetOutputInfos() { + return output_infos_; +} + TensorInfo OpenVINOBackend::GetOutputInfo(int index) { FDASSERT(index < NumOutputs(), "The index: %d should less than the number of outputs: %d.", index, @@ -181,19 +196,25 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file, // So here will reorder it's inputs and outputs std::string model_content; ReadBinaryFromFile(model_file, &model_content); - auto reader = paddle2onnx::OnnxReader(model_content.c_str(), model_content.size()); + auto reader = + paddle2onnx::OnnxReader(model_content.c_str(), model_content.size()); if (reader.num_inputs != input_infos.size()) { - FDERROR << "The number of inputs from OnnxReader:" << reader.num_inputs << " not equal to the number of inputs from OpenVINO:" << input_infos.size() << "." << std::endl; + FDERROR << "The number of inputs from OnnxReader:" << reader.num_inputs + << " not equal to the number of inputs from OpenVINO:" + << input_infos.size() << "." << std::endl; return false; } if (reader.num_outputs != output_infos.size()) { - FDERROR << "The number of outputs from OnnxReader:" << reader.num_outputs << " not equal to the number of outputs from OpenVINO:" << output_infos.size() << "." << std::endl; + FDERROR << "The number of outputs from OnnxReader:" << reader.num_outputs + << " not equal to the number of outputs from OpenVINO:" + << output_infos.size() << "." << std::endl; return false; } for (int i = 0; i < reader.num_inputs; ++i) { auto iter = input_infos.find(std::string(reader.inputs[i].name)); if (iter == input_infos.end()) { - FDERROR << "Cannot find input name:" << reader.inputs[i].name << " from OpenVINO model." << std::endl; + FDERROR << "Cannot find input name:" << reader.inputs[i].name + << " from OpenVINO model." << std::endl; return false; } input_infos_.push_back(iter->second); @@ -201,7 +222,8 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file, for (int i = 0; i < reader.num_outputs; ++i) { auto iter = output_infos.find(std::string(reader.outputs[i].name)); if (iter == output_infos.end()) { - FDERROR << "Cannot find output name:" << reader.outputs[i].name << " from OpenVINO model." << std::endl; + FDERROR << "Cannot find output name:" << reader.outputs[i].name + << " from OpenVINO model." << std::endl; return false; } output_infos_.push_back(iter->second); diff --git a/fastdeploy/backends/openvino/ov_backend.h b/fastdeploy/backends/openvino/ov_backend.h index fc25472f1a..5dd362d52a 100644 --- a/fastdeploy/backends/openvino/ov_backend.h +++ b/fastdeploy/backends/openvino/ov_backend.h @@ -51,6 +51,8 @@ class OpenVINOBackend : public BaseBackend { TensorInfo GetInputInfo(int index) override; TensorInfo GetOutputInfo(int index) override; + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; private: void InitTensorInfo(const std::vector>& ov_outputs, diff --git a/fastdeploy/backends/ort/ort_backend.cc b/fastdeploy/backends/ort/ort_backend.cc index 4615e9a38b..fba3793999 100644 --- a/fastdeploy/backends/ort/ort_backend.cc +++ b/fastdeploy/backends/ort/ort_backend.cc @@ -247,6 +247,16 @@ TensorInfo OrtBackend::GetInputInfo(int index) { return info; } +std::vector OrtBackend::GetInputInfos() { + auto size = inputs_desc_.size(); + std::vector infos; + infos.reserve(size); + for (auto i = 0; i < size; i++) { + infos.emplace_back(GetInputInfo(i)); + } + return infos; +} + TensorInfo OrtBackend::GetOutputInfo(int index) { FDASSERT(index < NumOutputs(), "The index: %d should less than the number of outputs: %d.", index, @@ -259,6 +269,14 @@ TensorInfo OrtBackend::GetOutputInfo(int index) { return info; } +std::vector OrtBackend::GetOutputInfos() { + std::vector infos; + for (auto i = 0; i < outputs_desc_.size(); i++) { + infos.emplace_back(GetOutputInfo(i)); + } + return infos; +} + void OrtBackend::InitCustomOperators() { #ifndef NON_64_PLATFORM if (custom_operators_.size() == 0) { diff --git a/fastdeploy/backends/ort/ort_backend.h b/fastdeploy/backends/ort/ort_backend.h index 526e28fc71..10c841ab29 100644 --- a/fastdeploy/backends/ort/ort_backend.h +++ b/fastdeploy/backends/ort/ort_backend.h @@ -74,6 +74,8 @@ class OrtBackend : public BaseBackend { TensorInfo GetInputInfo(int index); TensorInfo GetOutputInfo(int index); + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; static std::vector custom_operators_; void InitCustomOperators(); diff --git a/fastdeploy/backends/paddle/paddle_backend.cc b/fastdeploy/backends/paddle/paddle_backend.cc index 9f1b5bb8cc..831aae0a81 100644 --- a/fastdeploy/backends/paddle/paddle_backend.cc +++ b/fastdeploy/backends/paddle/paddle_backend.cc @@ -85,7 +85,7 @@ TensorInfo PaddleBackend::GetInputInfo(int index) { return inputs_desc_[index]; } -std::vector PaddleBackend::GetInputInfo() { return inputs_desc_; } +std::vector PaddleBackend::GetInputInfos() { return inputs_desc_; } TensorInfo PaddleBackend::GetOutputInfo(int index) { FDASSERT(index < NumOutputs(), @@ -94,7 +94,9 @@ TensorInfo PaddleBackend::GetOutputInfo(int index) { return outputs_desc_[index]; } -std::vector PaddleBackend::GetOutputInfo() { return outputs_desc_; } +std::vector PaddleBackend::GetOutputInfos() { + return outputs_desc_; +} bool PaddleBackend::Infer(std::vector& inputs, std::vector* outputs) { diff --git a/fastdeploy/backends/paddle/paddle_backend.h b/fastdeploy/backends/paddle/paddle_backend.h index 7ce82c9b33..ce4b33d7c4 100644 --- a/fastdeploy/backends/paddle/paddle_backend.h +++ b/fastdeploy/backends/paddle/paddle_backend.h @@ -75,8 +75,8 @@ class PaddleBackend : public BaseBackend { TensorInfo GetInputInfo(int index); TensorInfo GetOutputInfo(int index); - std::vector GetInputInfo(); - std::vector GetOutputInfo(); + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; private: paddle_infer::Config config_; diff --git a/fastdeploy/backends/tensorrt/trt_backend.cc b/fastdeploy/backends/tensorrt/trt_backend.cc index 841888f561..d8c336511b 100644 --- a/fastdeploy/backends/tensorrt/trt_backend.cc +++ b/fastdeploy/backends/tensorrt/trt_backend.cc @@ -589,6 +589,14 @@ TensorInfo TrtBackend::GetInputInfo(int index) { return info; } +std::vector TrtBackend::GetInputInfos() { + std::vector infos; + for (auto i = 0; i < inputs_desc_.size(); i++) { + infos.emplace_back(GetInputInfo(i)); + } + return infos; +} + TensorInfo TrtBackend::GetOutputInfo(int index) { FDASSERT(index < NumOutputs(), "The index: %d should less than the number of outputs: %d.", index, @@ -600,4 +608,13 @@ TensorInfo TrtBackend::GetOutputInfo(int index) { info.dtype = GetFDDataType(outputs_desc_[index].dtype); return info; } + +std::vector TrtBackend::GetOutputInfos() { + std::vector infos; + for (auto i = 0; i < outputs_desc_.size(); i++) { + infos.emplace_back(GetOutputInfo(i)); + } + return infos; +} + } // namespace fastdeploy diff --git a/fastdeploy/backends/tensorrt/trt_backend.h b/fastdeploy/backends/tensorrt/trt_backend.h index ed249a2a76..82b43ab46c 100644 --- a/fastdeploy/backends/tensorrt/trt_backend.h +++ b/fastdeploy/backends/tensorrt/trt_backend.h @@ -72,6 +72,8 @@ class TrtBackend : public BaseBackend { int NumOutputs() const { return outputs_desc_.size(); } TensorInfo GetInputInfo(int index); TensorInfo GetOutputInfo(int index); + std::vector GetInputInfos() override; + std::vector GetOutputInfos() override; ~TrtBackend() { if (parser_) { diff --git a/fastdeploy/core/fd_tensor.cc b/fastdeploy/core/fd_tensor.cc index 9b5dbd0a44..e988b60daf 100644 --- a/fastdeploy/core/fd_tensor.cc +++ b/fastdeploy/core/fd_tensor.cc @@ -188,5 +188,33 @@ void FDTensor::PrintInfo(const std::string& prefix) { << ", min=" << min << std::endl; } +bool FDTensor::AllocFn(size_t nbytes) { + if (device == Device::GPU) { +#ifdef WITH_GPU + return FDDeviceAllocator()(&buffer_, nbytes); +#else + FDASSERT(false, + "The FastDeploy FDTensor allocator didn't compile under " + "-DWITH_GPU=ON," + "so this is an unexpected problem happend."); +#endif + } + return FDHostAllocator()(&buffer_, nbytes); +} + +void FDTensor::FreeFn() { + if (external_data_ptr != nullptr) external_data_ptr = nullptr; + if (buffer_ != nullptr) { + if (device == Device::GPU) { +#ifdef WITH_GPU + FDDeviceFree()(buffer_); +#endif + } else { + FDHostFree()(buffer_); + } + buffer_ = nullptr; + } +} + FDTensor::FDTensor(const std::string& tensor_name) { name = tensor_name; } } // namespace fastdeploy diff --git a/fastdeploy/core/fd_tensor.h b/fastdeploy/core/fd_tensor.h index ab52b58dc8..6801380064 100644 --- a/fastdeploy/core/fd_tensor.h +++ b/fastdeploy/core/fd_tensor.h @@ -98,33 +98,9 @@ struct FASTDEPLOY_DECL FDTensor { // prefix will also be printed as tag void PrintInfo(const std::string& prefix = "TensorInfo: "); - bool AllocFn(size_t nbytes) { - if (device == Device::GPU) { -#ifdef WITH_GPU - return FDDeviceAllocator()(&buffer_, nbytes); -#else - FDASSERT(false, - "The FastDeploy FDTensor allocator didn't compile under " - "-DWITH_GPU=ON," - "so this is an unexpected problem happend."); -#endif - } - return FDHostAllocator()(&buffer_, nbytes); - } - - void FreeFn() { - if (external_data_ptr != nullptr) external_data_ptr = nullptr; - if (buffer_ != nullptr) { - if (device == Device::GPU) { -#ifdef WITH_GPU - FDDeviceFree()(buffer_); -#endif - } else { - FDHostFree()(buffer_); - } - buffer_ = nullptr; - } - } + bool AllocFn(size_t nbytes); + + void FreeFn(); FDTensor() {} explicit FDTensor(const std::string& tensor_name); diff --git a/fastdeploy/fastdeploy_runtime.cc b/fastdeploy/fastdeploy_runtime.cc index 90369ada59..3c119f7ab1 100644 --- a/fastdeploy/fastdeploy_runtime.cc +++ b/fastdeploy/fastdeploy_runtime.cc @@ -13,6 +13,7 @@ // limitations under the License. #include "fastdeploy/fastdeploy_runtime.h" + #include "fastdeploy/utils/unique_ptr.h" #include "fastdeploy/utils/utils.h" @@ -297,6 +298,14 @@ TensorInfo Runtime::GetOutputInfo(int index) { return backend_->GetOutputInfo(index); } +std::vector Runtime::GetInputInfos() { + return backend_->GetInputInfos(); +} + +std::vector Runtime::GetOutputInfos() { + return backend_->GetOutputInfos(); +} + bool Runtime::Infer(std::vector& input_tensors, std::vector* output_tensors) { return backend_->Infer(input_tensors, output_tensors); diff --git a/fastdeploy/fastdeploy_runtime.h b/fastdeploy/fastdeploy_runtime.h index a64ac375d7..58981687d1 100644 --- a/fastdeploy/fastdeploy_runtime.h +++ b/fastdeploy/fastdeploy_runtime.h @@ -165,6 +165,8 @@ struct FASTDEPLOY_DECL Runtime { int NumOutputs() { return backend_->NumOutputs(); } TensorInfo GetInputInfo(int index); TensorInfo GetOutputInfo(int index); + std::vector GetInputInfos(); + std::vector GetOutputInfos(); RuntimeOption option;