[CVCUDA] Vision Processor Python API and Tutorial (PaddlePaddle#1394)

* bind success

* bind success fix

* FDMat pybind, ResizeByShort pybind

* FDMat pybind, ResizeByShort pybind, remove initialized_

* override BindProcessorManager::Run in python is available

* PyProcessorManager done

* vision_pybind fix

* manager.py fix

* add tutorials

* remove Apply() bind

* remove Apply() bind and fix

* fix reviewed problem

* fix reviewed problem

* fix reviewed problem readme

* fix reviewed problem readme etc

* apply return outputs

* nits

* update readme

* fix FDMatbatch

* add op pybind: CenterCrop, Pad

* add op overload for pass FDMatBatch

---------

Co-authored-by: Wang Xinyu <[email protected]>

fastdeploy/vision/common/processors/base_pybind.cc

@@ -0,0 +1,28 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include <pybind11/operators.h>
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindProcessor(pybind11::module& m) {
+  pybind11::class_<vision::Processor>(m, "Processor")
+      .def("__call__", [](vision::Processor& self,
+                          vision::FDMat* mat) { return self(mat); })
+      .def("__call__",
+           [](vision::Processor& self, vision::FDMatBatch* mat_batch) {
+             return self(mat_batch);
+           });
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/center_crop_pybind.cc

@@ -0,0 +1,23 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindCenterCrop(pybind11::module& m) {
+  pybind11::class_<vision::CenterCrop, vision::Processor>(
+      m, "CenterCrop")
+      .def(pybind11::init<int, int>(), "Default constructor");
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/manager.cc

@@ -49,48 +49,51 @@ bool ProcessorManager::CudaUsed() {
   return (proc_lib_ == ProcLib::CUDA || proc_lib_ == ProcLib::CVCUDA);
 }
 
-bool ProcessorManager::Run(std::vector<FDMat>* images,
-                           std::vector<FDTensor>* outputs) {
-  if (images->size() == 0) {
-    FDERROR << "The size of input images should be greater than 0."
-            << std::endl;
-    return false;
-  }
+void ProcessorManager::PreApply(FDMatBatch* image_batch) {
+  FDASSERT(image_batch->mats != nullptr, "The mats is empty.");
+  FDASSERT(image_batch->mats->size() > 0,
+           "The size of input images should be greater than 0.");
 
-  if (images->size() > input_caches_.size()) {
-    input_caches_.resize(images->size());
-    output_caches_.resize(images->size());
+  if (image_batch->mats->size() > input_caches_.size()) {
+    input_caches_.resize(image_batch->mats->size());
+    output_caches_.resize(image_batch->mats->size());
   }
+  image_batch->input_cache = &batch_input_cache_;
+  image_batch->output_cache = &batch_output_cache_;
 
-  FDMatBatch image_batch(images);
-  image_batch.input_cache = &batch_input_cache_;
-  image_batch.output_cache = &batch_output_cache_;
-  image_batch.proc_lib = proc_lib_;
+  if (CudaUsed()) {
+    SetStream(image_batch);
+  }
 
-  for (size_t i = 0; i < images->size(); ++i) {
-    if (CudaUsed()) {
-      SetStream(&image_batch);
-    }
-    (*images)[i].input_cache = &input_caches_[i];
-    (*images)[i].output_cache = &output_caches_[i];
-    (*images)[i].proc_lib = proc_lib_;
-    if ((*images)[i].mat_type == ProcLib::CUDA) {
+  for (size_t i = 0; i < image_batch->mats->size(); ++i) {
+    FDMat* mat = &(image_batch->mats->at(i));
+    mat->input_cache = &input_caches_[i];
+    mat->output_cache = &output_caches_[i];
+    mat->proc_lib = proc_lib_;
+    if (mat->mat_type == ProcLib::CUDA) {
       // Make a copy of the input data ptr, so that the original data ptr of
       // FDMat won't be modified.
       auto fd_tensor = std::make_shared<FDTensor>();
-      fd_tensor->SetExternalData(
-          (*images)[i].Tensor()->shape, (*images)[i].Tensor()->Dtype(),
-          (*images)[i].Tensor()->Data(), (*images)[i].Tensor()->device,
-          (*images)[i].Tensor()->device_id);
-      (*images)[i].SetTensor(fd_tensor);
+      fd_tensor->SetExternalData(mat->Tensor()->shape, mat->Tensor()->Dtype(),
+                                 mat->Tensor()->Data(), mat->Tensor()->device,
+                                 mat->Tensor()->device_id);
+      mat->SetTensor(fd_tensor);
     }
   }
+}
 
-  bool ret = Apply(&image_batch, outputs);
-
+void ProcessorManager::PostApply() {
   if (CudaUsed()) {
     SyncStream();
   }
+}
+
+bool ProcessorManager::Run(std::vector<FDMat>* images,
+                           std::vector<FDTensor>* outputs) {
+  FDMatBatch image_batch(images);
+  PreApply(&image_batch);
+  bool ret = Apply(&image_batch, outputs);
+  PostApply();
   return ret;
 }
 

fastdeploy/vision/common/processors/manager.h

@@ -78,6 +78,10 @@ class FASTDEPLOY_DECL ProcessorManager {
   virtual bool Apply(FDMatBatch* image_batch,
                      std::vector<FDTensor>* outputs) = 0;
 
+  void PreApply(FDMatBatch* image_batch);
+
+  void PostApply();
+
  protected:
   ProcLib proc_lib_ = ProcLib::DEFAULT;
 

fastdeploy/vision/common/processors/manager_pybind.cc

@@ -14,8 +14,22 @@
 #include "fastdeploy/pybind/main.h"
 
 namespace fastdeploy {
+namespace vision {
+// PyProcessorManager is used for pybind11::init() of ProcessorManager
+// Because ProcessorManager have a pure Virtual function Apply()
+class FASTDEPLOY_DECL PyProcessorManager : public ProcessorManager {
+ public:
+  using ProcessorManager::ProcessorManager;
+  bool Apply(FDMatBatch* image_batch, std::vector<FDTensor>* outputs) override {
+    PYBIND11_OVERRIDE_PURE(bool, ProcessorManager, Apply, image_batch, outputs);
+  }
+};
+}  // namespace vision
+
 void BindProcessorManager(pybind11::module& m) {
-  pybind11::class_<vision::ProcessorManager>(m, "ProcessorManager")
+  pybind11::class_<vision::ProcessorManager, vision::PyProcessorManager>(
+      m, "ProcessorManager")
+      .def(pybind11::init<>())
       .def("run",
            [](vision::ProcessorManager& self,
               std::vector<pybind11::array>& im_list) {
@@ -34,6 +48,8 @@ void BindProcessorManager(pybind11::module& m) {
              }
              return outputs;
            })
+      .def("pre_apply", &vision::ProcessorManager::PreApply)
+      .def("post_apply", &vision::ProcessorManager::PostApply)
       .def("use_cuda",
            [](vision::ProcessorManager& self, bool enable_cv_cuda = false,
               int gpu_id = -1) { self.UseCuda(enable_cv_cuda, gpu_id); });

fastdeploy/vision/common/processors/mat_batch.h

@@ -57,6 +57,10 @@ struct FASTDEPLOY_DECL FDMatBatch {
 #endif
 
   std::vector<FDMat>* mats = nullptr;
+
+  // Used by pybind, since python cannot pass list as pointer or reference
+  std::vector<FDMat> mats_holder;
+
   ProcLib mat_type = ProcLib::OPENCV;
   FDMatBatchLayout layout = FDMatBatchLayout::NHWC;
   Device device = Device::CPU;

fastdeploy/vision/common/processors/mat_batch_pybind.cc

@@ -0,0 +1,30 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindFDMatBatch(pybind11::module& m) {
+  pybind11::class_<vision::FDMatBatch>(m, "FDMatBatch")
+      .def(pybind11::init<>(), "Default constructor")
+      .def_readwrite("input_cache", &vision::FDMatBatch::input_cache)
+      .def_readwrite("output_cache", &vision::FDMatBatch::output_cache)
+      .def_readwrite("mats", &vision::FDMatBatch::mats)
+      .def("from_mats",
+             [](vision::FDMatBatch& self, std::vector<vision::FDMat>& _mats) {
+                  self.mats_holder = _mats;
+                  self.mats = &(self.mats_holder);
+             });
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/mat_pybind.cc

@@ -0,0 +1,29 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindFDMat(pybind11::module& m) {
+  pybind11::class_<vision::FDMat>(m, "FDMat")
+      .def(pybind11::init<>(), "Default constructor")
+      .def_readwrite("input_cache", &vision::FDMat::input_cache)
+      .def_readwrite("output_cache", &vision::FDMat::output_cache)
+      .def("from_numpy",
+             [](vision::FDMat& self, pybind11::array& pyarray) {
+                self = vision::WrapMat(PyArrayToCvMat(pyarray));
+             })
+      .def("print_info", &vision::FDMat::PrintInfo);
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/normalize_and_permute_pybind.cc

@@ -0,0 +1,25 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindNormalizeAndPermute(pybind11::module& m) {
+  pybind11::class_<vision::NormalizeAndPermute, vision::Processor>(
+      m, "NormalizeAndPermute")
+      .def(pybind11::init<std::vector<float>, std::vector<float>, bool,
+                          std::vector<float>, std::vector<float>, bool>(),
+           "Default constructor");
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/pad_pybind.cc

@@ -0,0 +1,23 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindPad(pybind11::module& m) {
+  pybind11::class_<vision::Pad, vision::Processor>(
+      m, "Pad")
+      .def(pybind11::init<int, int, int, int, std::vector<float>>(), "Default constructor");
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/processors_pybind.cc

@@ -0,0 +1,36 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+
+void BindProcessorManager(pybind11::module& m);
+void BindNormalizeAndPermute(pybind11::module& m);
+void BindProcessor(pybind11::module& m);
+void BindResizeByShort(pybind11::module& m);
+void BindCenterCrop(pybind11::module& m);
+void BindPad(pybind11::module& m);
+
+void BindProcessors(pybind11::module& m) {
+  auto processors_m =
+      m.def_submodule("processors", "Module to deploy Processors models");
+  BindProcessorManager(processors_m);
+  BindProcessor(processors_m);
+  BindNormalizeAndPermute(processors_m);
+  BindResizeByShort(processors_m);
+  BindCenterCrop(processors_m);
+  BindPad(processors_m);
+}
+}  // namespace fastdeploy

fastdeploy/vision/common/processors/resize_by_short_pybind.cc

@@ -0,0 +1,23 @@
+// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "fastdeploy/pybind/main.h"
+
+namespace fastdeploy {
+void BindResizeByShort(pybind11::module& m) {
+  pybind11::class_<vision::ResizeByShort, vision::Processor>(m, "ResizeByShort")
+      .def(pybind11::init<int, int, bool, std::vector<int>>(),
+           "Default constructor");
+}
+
+}  // namespace fastdeploy

fastdeploy/vision/vision_pybind.cc

@@ -16,7 +16,9 @@
 
 namespace fastdeploy {
 
-void BindProcessorManager(pybind11::module& m);
+void BindFDMat(pybind11::module& m);
+void BindFDMatBatch(pybind11::module& m);
+void BindProcessors(pybind11::module& m);
 void BindDetection(pybind11::module& m);
 void BindClassification(pybind11::module& m);
 void BindSegmentation(pybind11::module& m);
@@ -205,7 +207,9 @@ void BindVision(pybind11::module& m) {
   m.def("disable_flycv", &vision::DisableFlyCV,
         "Disable image preprocessing by FlyCV, change to use OpenCV.");
 
-  BindProcessorManager(m);
+  BindFDMat(m);
+  BindFDMatBatch(m);
+  BindProcessors(m);
   BindDetection(m);
   BindClassification(m);
   BindSegmentation(m);

python/fastdeploy/vision/common/__init__.py

@@ -14,3 +14,4 @@
 from __future__ import absolute_import
 
 from .manager import ProcessorManager
+from .manager import PyProcessorManager