[Example] Update runtime examples (PaddlePaddle#1542)

* Add notes for tensors * Optimize some apis * move some warnings
Petal99 · Mar 8, 2023 · 6be2c03 · 6be2c03
1 parent 3d31834
commit 6be2c03
Show file tree

Hide file tree

Showing 17 changed files with 410 additions and 262 deletions.
diff --git a/examples/runtime/cpp/CMakeLists.txt b/examples/runtime/cpp/CMakeLists.txt
@@ -9,6 +9,6 @@ include(${FASTDEPLOY_INSTALL_DIR}/FastDeploy.cmake)
 # 添加FastDeploy依赖头文件
 include_directories(${FASTDEPLOY_INCS})
 
-add_executable(runtime_demo ${PROJECT_SOURCE_DIR}/infer_onnx_openvino.cc)
+add_executable(runtime_demo ${PROJECT_SOURCE_DIR}/infer_paddle_paddle_inference.cc)
 # 添加FastDeploy库依赖
 target_link_libraries(runtime_demo ${FASTDEPLOY_LIBS})
diff --git a/examples/runtime/cpp/infer_onnx_onnxruntime.cc b/examples/runtime/cpp/infer_onnx_onnxruntime.cc
@@ -13,52 +13,60 @@
 // limitations under the License.
 
 #include "fastdeploy/runtime.h"
+#include <cassert>
 
 namespace fd = fastdeploy;
 
 int main(int argc, char* argv[]) {
-  std::string model_file = "mobilenetv2.onnx";
+  // Download from https://bj.bcebos.com/paddle2onnx/model_zoo/pplcnet.onnx
+  std::string model_file = "pplcnet.onnx";
 
-  // setup option
+  // configure runtime
+  // How to configure by RuntimeOption, refer its api doc for more information
+  // https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1RuntimeOption.html
   fd::RuntimeOption runtime_option;
   runtime_option.SetModelPath(model_file, "", fd::ModelFormat::ONNX);
   runtime_option.UseOrtBackend();
+
+  // Use CPU to inference
+  runtime_option.UseCpu();
   runtime_option.SetCpuThreadNum(12);
 
-  // **** GPU ****
-  // To use GPU, use the following commented code
+  // Use Gpu to inference
   // runtime_option.UseGpu(0);
+  // If need to configure ONNX Runtime backend for more option, we can configure runtime_option.ort_option
+  // refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1OrtBackendOption.html
 
-  // init runtime
-  std::unique_ptr<fd::Runtime> runtime =
-      std::unique_ptr<fd::Runtime>(new fd::Runtime());
-  if (!runtime->Init(runtime_option)) {
-    std::cerr << "--- Init FastDeploy Runitme Failed! "
-              << "\n--- Model:  " << model_file << std::endl;
-    return -1;
-  } else {
-    std::cout << "--- Init FastDeploy Runitme Done! "
-              << "\n--- Model:  " << model_file << std::endl;
-  }
-  // init input tensor shape
-  fd::TensorInfo info = runtime->GetInputInfo(0);
-  info.shape = {1, 3, 224, 224};
+  fd::Runtime runtime;
+  assert(runtime.Init(runtime_option));
 
-  std::vector<fd::FDTensor> input_tensors(1);
-  std::vector<fd::FDTensor> output_tensors(1);
+  // Get model's inputs information
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1Runtime.html
+  std::vector<fd::TensorInfo> inputs_info = runtime.GetInputInfos();
 
-  std::vector<float> inputs_data;
-  inputs_data.resize(1 * 3 * 224 * 224);
-  for (size_t i = 0; i < inputs_data.size(); ++i) {
-    inputs_data[i] = std::rand() % 1000 / 1000.0f;
-  }
-  input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data());
-
-  //get input name
-  input_tensors[0].name = info.name;
+  // Create dummy data fill with 0.5
+  std::vector<float> dummy_data(1 * 3 * 224 * 224, 0.5);
 
-  runtime->Infer(input_tensors, &output_tensors);
+  // Create inputs/outputs tensors
+  std::vector<fd::FDTensor> inputs(inputs_info.size());
+  std::vector<fd::FDTensor> outputs;
 
-  output_tensors[0].PrintInfo();
+  // Initialize input tensors
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1FDTensor.html
+  inputs[0].SetData({1, 3, 224, 224}, fd::FDDataType::FP32, dummy_data.data());
+  inputs[0].name = inputs_info[0].name;
+
+  // Inference
+  assert(runtime.Infer(inputs, &outputs));
+
+  // Print debug information of outputs 
+  outputs[0].PrintInfo();
+
+  // Get data pointer and print it's elements
+  const float* data_ptr = reinterpret_cast<const float*>(outputs[0].GetData());
+  for (size_t i = 0; i < 10 && i < outputs[0].Numel(); ++i) {
+    std::cout << data_ptr[i] << " ";
+  }
+  std::cout << std::endl;
   return 0;
-}
+}
diff --git a/examples/runtime/cpp/infer_onnx_openvino.cc b/examples/runtime/cpp/infer_onnx_openvino.cc
@@ -13,47 +13,57 @@
 // limitations under the License.
 
 #include "fastdeploy/runtime.h"
+#include <cassert>
 
 namespace fd = fastdeploy;
 
 int main(int argc, char* argv[]) {
-  std::string model_file = "mobilenetv2.onnx";
+  // Download from https://bj.bcebos.com/paddle2onnx/model_zoo/pplcnet.onnx
+  std::string model_file = "pplcnet.onnx";
 
-  // setup option
+  // configure runtime
+  // How to configure by RuntimeOption, refer its api doc for more information
+  // https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1RuntimeOption.html
   fd::RuntimeOption runtime_option;
   runtime_option.SetModelPath(model_file, "", fd::ModelFormat::ONNX);
   runtime_option.UseOpenVINOBackend();
+
+  // Use CPU to inference
+  // If need to configure OpenVINO backend for more option, we can configure runtime_option.openvino_option
+  // refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1OpenVINOBackendOption.html
+  runtime_option.UseCpu();
   runtime_option.SetCpuThreadNum(12);
-  // init runtime
-  std::unique_ptr<fd::Runtime> runtime =
-      std::unique_ptr<fd::Runtime>(new fd::Runtime());
-  if (!runtime->Init(runtime_option)) {
-    std::cerr << "--- Init FastDeploy Runitme Failed! "
-              << "\n--- Model:  " << model_file << std::endl;
-    return -1;
-  } else {
-    std::cout << "--- Init FastDeploy Runitme Done! "
-              << "\n--- Model:  " << model_file << std::endl;
-  }
-  // init input tensor shape
-  fd::TensorInfo info = runtime->GetInputInfo(0);
-  info.shape = {1, 3, 224, 224};
 
-  std::vector<fd::FDTensor> input_tensors(1);
-  std::vector<fd::FDTensor> output_tensors(1);
+  fd::Runtime runtime;
+  assert(runtime.Init(runtime_option));
 
-  std::vector<float> inputs_data;
-  inputs_data.resize(1 * 3 * 224 * 224);
-  for (size_t i = 0; i < inputs_data.size(); ++i) {
-    inputs_data[i] = std::rand() % 1000 / 1000.0f;
-  }
-  input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data());
-
-  //get input name
-  input_tensors[0].name = info.name;
+  // Get model's inputs information
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1Runtime.html
+  std::vector<fd::TensorInfo> inputs_info = runtime.GetInputInfos();
 
-  runtime->Infer(input_tensors, &output_tensors);
+  // Create dummy data fill with 0.5
+  std::vector<float> dummy_data(1 * 3 * 224 * 224, 0.5);
 
-  output_tensors[0].PrintInfo();
+  // Create inputs/outputs tensors
+  std::vector<fd::FDTensor> inputs(inputs_info.size());
+  std::vector<fd::FDTensor> outputs;
+
+  // Initialize input tensors
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1FDTensor.html
+  inputs[0].SetData({1, 3, 224, 224}, fd::FDDataType::FP32, dummy_data.data());
+  inputs[0].name = inputs_info[0].name;
+
+  // Inference
+  assert(runtime.Infer(inputs, &outputs));
+
+  // Print debug information of outputs 
+  outputs[0].PrintInfo();
+
+  // Get data pointer and print it's elements
+  const float* data_ptr = reinterpret_cast<const float*>(outputs[0].GetData());
+  for (size_t i = 0; i < 10 && i < outputs[0].Numel(); ++i) {
+    std::cout << data_ptr[i] << " ";
+  }
+  std::cout << std::endl;
   return 0;
-}
+}
diff --git a/examples/runtime/cpp/infer_onnx_tensorrt.cc b/examples/runtime/cpp/infer_onnx_tensorrt.cc
@@ -13,48 +13,60 @@
 // limitations under the License.
 
 #include "fastdeploy/runtime.h"
+#include <cassert>
 
 namespace fd = fastdeploy;
 
 int main(int argc, char* argv[]) {
-  std::string model_file = "mobilenetv2.onnx";
+  // Download from https://bj.bcebos.com/paddle2onnx/model_zoo/pplcnet.onnx
+  std::string model_file = "pplcnet.onnx";
 
-  // setup option
+  // configure runtime
+  // How to configure by RuntimeOption, refer its api doc for more information
+  // https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1RuntimeOption.html
   fd::RuntimeOption runtime_option;
   runtime_option.SetModelPath(model_file, "", fd::ModelFormat::ONNX);
-  runtime_option.UseGpu(0);
   runtime_option.UseTrtBackend();
-  runtime_option.SetTrtInputShape("inputs", {1, 3, 224, 224});
-  // init runtime
-  std::unique_ptr<fd::Runtime> runtime =
-      std::unique_ptr<fd::Runtime>(new fd::Runtime());
-  if (!runtime->Init(runtime_option)) {
-    std::cerr << "--- Init FastDeploy Runitme Failed! "
-              << "\n--- Model:  " << model_file << std::endl;
-    return -1;
-  } else {
-    std::cout << "--- Init FastDeploy Runitme Done! "
-              << "\n--- Model:  " << model_file << std::endl;
-  }
-  // init input tensor shape
-  fd::TensorInfo info = runtime->GetInputInfo(0);
-  info.shape = {1, 3, 224, 224};
+
+  // Use NVIDIA GPU to inference
+  // If need to configure TensorRT backend for more option, we can configure runtime_option.trt_option
+  // refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1TrtBackendOption.html
+  runtime_option.UseGpu(0);
+  // Use float16 inference to improve performance
+  runtime_option.trt_option.enable_fp16 = true;
+  // Cache trt engine to reduce time cost in model initialize
+  runtime_option.trt_option.serialize_file = "./model.trt";
 
-  std::vector<fd::FDTensor> input_tensors(1);
-  std::vector<fd::FDTensor> output_tensors(1);
+  fd::Runtime runtime;
+  assert(runtime.Init(runtime_option));
 
-  std::vector<float> inputs_data;
-  inputs_data.resize(1 * 3 * 224 * 224);
-  for (size_t i = 0; i < inputs_data.size(); ++i) {
-    inputs_data[i] = std::rand() % 1000 / 1000.0f;
-  }
-  input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data());
-
-  //get input name
-  input_tensors[0].name = info.name;
+  // Get model's inputs information
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1Runtime.html
+  std::vector<fd::TensorInfo> inputs_info = runtime.GetInputInfos();
+
+  // Create dummy data fill with 0.5
+  std::vector<float> dummy_data(1 * 3 * 224 * 224, 0.5);
+
+  // Create inputs/outputs tensors
+  std::vector<fd::FDTensor> inputs(inputs_info.size());
+  std::vector<fd::FDTensor> outputs;
 
-  runtime->Infer(input_tensors, &output_tensors);
+  // Initialize input tensors
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1FDTensor.html
+  inputs[0].SetData({1, 3, 224, 224}, fd::FDDataType::FP32, dummy_data.data());
+  inputs[0].name = inputs_info[0].name;
 
-  output_tensors[0].PrintInfo();
+  // Inference
+  assert(runtime.Infer(inputs, &outputs));
+
+  // Print debug information of outputs 
+  outputs[0].PrintInfo();
+
+  // Get data pointer and print it's elements
+  const float* data_ptr = reinterpret_cast<const float*>(outputs[0].GetData());
+  for (size_t i = 0; i < 10 && i < outputs[0].Numel(); ++i) {
+    std::cout << data_ptr[i] << " ";
+  }
+  std::cout << std::endl;
   return 0;
-}
+}
diff --git a/examples/runtime/cpp/infer_paddle_onnxruntime.cc b/examples/runtime/cpp/infer_paddle_onnxruntime.cc
@@ -13,53 +13,61 @@
 // limitations under the License.
 
 #include "fastdeploy/runtime.h"
+#include <cassert>
 
 namespace fd = fastdeploy;
 
 int main(int argc, char* argv[]) {
-  std::string model_file = "mobilenetv2/inference.pdmodel";
-  std::string params_file = "mobilenetv2/inference.pdiparams";
+  // Download from https://bj.bcebos.com/paddle2onnx/model_zoo/pplcnet.tar.gz
+  std::string model_file = "pplcnet/inference.pdmodel";
+  std::string params_file = "pplcnet/inference.pdiparams";
 
-  // setup option
+  // configure runtime
+  // How to configure by RuntimeOption, refer its api doc for more information
+  // https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1RuntimeOption.html
   fd::RuntimeOption runtime_option;
-  runtime_option.SetModelPath(model_file, params_file, fd::ModelFormat::PADDLE);
+  runtime_option.SetModelPath(model_file, params_file);
   runtime_option.UseOrtBackend();
+
+  // Use CPU to inference
+  runtime_option.UseCpu();
   runtime_option.SetCpuThreadNum(12);
 
-  // **** GPU ****
-  // To use GPU, use the following commented code
+  // Use Gpu to inference
   // runtime_option.UseGpu(0);
+  // If need to configure ONNX Runtime backend for more option, we can configure runtime_option.ort_option
+  // refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1OrtBackendOption.html
 
-  // init runtime
-  std::unique_ptr<fd::Runtime> runtime =
-      std::unique_ptr<fd::Runtime>(new fd::Runtime());
-  if (!runtime->Init(runtime_option)) {
-    std::cerr << "--- Init FastDeploy Runitme Failed! "
-              << "\n--- Model:  " << model_file << std::endl;
-    return -1;
-  } else {
-    std::cout << "--- Init FastDeploy Runitme Done! "
-              << "\n--- Model:  " << model_file << std::endl;
-  }
-  // init input tensor shape
-  fd::TensorInfo info = runtime->GetInputInfo(0);
-  info.shape = {1, 3, 224, 224};
+  fd::Runtime runtime;
+  assert(runtime.Init(runtime_option));
 
-  std::vector<fd::FDTensor> input_tensors(1);
-  std::vector<fd::FDTensor> output_tensors(1);
+  // Get model's inputs information
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1Runtime.html
+  std::vector<fd::TensorInfo> inputs_info = runtime.GetInputInfos();
 
-  std::vector<float> inputs_data;
-  inputs_data.resize(1 * 3 * 224 * 224);
-  for (size_t i = 0; i < inputs_data.size(); ++i) {
-    inputs_data[i] = std::rand() % 1000 / 1000.0f;
-  }
-  input_tensors[0].SetExternalData({1, 3, 224, 224}, fd::FDDataType::FP32, inputs_data.data());
-
-  //get input name
-  input_tensors[0].name = info.name;
+  // Create dummy data fill with 0.5
+  std::vector<float> dummy_data(1 * 3 * 224 * 224, 0.5);
 
-  runtime->Infer(input_tensors, &output_tensors);
+  // Create inputs/outputs tensors
+  std::vector<fd::FDTensor> inputs(inputs_info.size());
+  std::vector<fd::FDTensor> outputs;
 
-  output_tensors[0].PrintInfo();
+  // Initialize input tensors
+  // API doc refer https://baidu-paddle.github.io/fastdeploy-api/cpp/html/structfastdeploy_1_1FDTensor.html
+  inputs[0].SetData({1, 3, 224, 224}, fd::FDDataType::FP32, dummy_data.data());
+  inputs[0].name = inputs_info[0].name;
+
+  // Inference
+  assert(runtime.Infer(inputs, &outputs));
+
+  // Print debug information of outputs 
+  outputs[0].PrintInfo();
+
+  // Get data pointer and print it's elements
+  const float* data_ptr = reinterpret_cast<const float*>(outputs[0].GetData());
+  for (size_t i = 0; i < 10 && i < outputs[0].Numel(); ++i) {
+    std::cout << data_ptr[i] << " ";
+  }
+  std::cout << std::endl;
   return 0;
-}
+}