Skip to content

Commit 988fc3b

Browse files
dkalinowskidkalinowski
authored
CVS-77193 DAG layout intersections & binary image handling changes (#1143)
- dag reports correct intersection of 2 layouts if 2 inputs are connected to the same pipeline input - binary input is accepted for DAGs/models with layouts which intersect correctly with N...HWC layout (for example N... (default), N?... (default demulti), ... (custom node), NHWC, N?HWC (demulti), NH..., N...C, etc) - binary input resize and multiple image alignment is performed only for layouts NHWC, N?HWC and ...
1 parent 73f3b61 commit 988fc3b

19 files changed

+562
-176
lines changed

‎docs/binary_input.md

+8-2
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,17 @@ is out of range it will be adjusted to the nearer border. For example, when mode
1919
- if input shape is [1,90,200,3] it will be resized into [1,100,200,3]
2020
- if input shape is [1,220,200,3] it will be resized into [1,200,200,3]
2121

22-
Processing the binary image requests requires the model or the custom nodes to accept NHWC layout in BGR color
22+
In order to use binary input functionality, model or pipeline input layout needs to be compatible with `N...HWC` and have 4 (or 5 in case of demultiplexing) shape dimensions. It means that input layout needs to resemble `NHWC` layout, e.g. default `N...` will work. On the other hand, binary image input is not supported for inputs with `NCHW` layout.
23+
24+
To fully utilize binary input utility, automatic image size alignment will be done by OVMS when:
25+
- input shape does not include dynamic dimension value (`-1`)
26+
- input layout is configured to be either `...` (custom nodes) and `NHWC` or `N?HWC` (or `N?HWC`, when modified by a [demultiplexer](demultiplexing.md))
27+
28+
Processing the binary image requests requires the model or the custom nodes to accept BGR color
2329
format with data with the data range from 0-255. Original layout of the input data can be changed in the
2430
OVMS configuration in runtime. For example when the orignal model has input shape [1,3,224,224] add a parameter
2531
in the OVMS configuration "layout": "NHWC:NCHW" or the command line parameter `--layout NHWC:NCHW`. In result, the model will
26-
have effective shape [1,224,224,3].
32+
have effective shape [1,224,224,3] and layout `NHWC`.
2733

2834
In case the model was trained with color format RGB and range other then 0-255, the
2935
[model optimizer](tf_model_binary_input.md)

‎src/binaryutils.cpp

+111-81
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ cv::Mat convertStringValToMat(const std::string& stringVal) {
7272
try {
7373
return cv::imdecode(dataMat, cv::IMREAD_UNCHANGED);
7474
} catch (const cv::Exception& e) {
75-
SPDLOG_ERROR("Error during string_val to mat conversion: {}", e.what());
75+
SPDLOG_DEBUG("Error during string_val to mat conversion: {}", e.what());
7676
return cv::Mat{};
7777
}
7878
}
@@ -89,79 +89,25 @@ Status convertPrecision(const cv::Mat& src, cv::Mat& dst, const ovms::Precision
8989
}
9090

9191
Status validateLayout(const std::shared_ptr<TensorInfo>& tensorInfo) {
92-
if ((tensorInfo->getLayout() != "NHWC") &&
93-
(tensorInfo->getLayout() != Layout::getUnspecifiedLayout()) && // handle DAG
94-
(tensorInfo->getLayout() != Layout::getDefaultLayout())) { // handle model without Layout set
92+
static const std::string binarySupportedLayout = "N...HWC";
93+
if (!tensorInfo->getLayout().createIntersection(Layout(binarySupportedLayout), tensorInfo->getShape().size()).has_value()) {
94+
SPDLOG_DEBUG("Endpoint needs to be compatible with {} to support binary image inputs, actual: {}",
95+
binarySupportedLayout,
96+
tensorInfo->getLayout());
9597
return StatusCode::UNSUPPORTED_LAYOUT;
9698
}
9799
return StatusCode::OK;
98100
}
99101

100-
bool resizeNeeded(const cv::Mat& image, const std::shared_ptr<TensorInfo>& tensorInfo) {
101-
Dimension cols = Dimension::any();
102-
Dimension rows = Dimension::any();
103-
if (tensorInfo->getShape().size() == 4) {
104-
cols = tensorInfo->getShape()[2];
105-
rows = tensorInfo->getShape()[1];
106-
} else if (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5) {
107-
cols = tensorInfo->getShape()[3];
108-
rows = tensorInfo->getShape()[2];
109-
} else {
110-
return false;
111-
}
112-
if (cols.isAny()) {
113-
cols = image.cols;
114-
}
115-
if (rows.isAny()) {
116-
rows = image.rows;
117-
}
118-
if ((!cols.match(image.cols)) || (!rows.match(image.rows))) {
102+
bool resizeNeeded(const cv::Mat& image, const dimension_value_t height, const dimension_value_t width) {
103+
if (height != image.rows || width != image.cols) {
119104
return true;
120105
}
121106
return false;
122107
}
123108

124-
Status resizeMat(const cv::Mat& src, cv::Mat& dst, const std::shared_ptr<TensorInfo>& tensorInfo) {
125-
Dimension cols = Dimension::any();
126-
Dimension rows = Dimension::any();
127-
if (tensorInfo->getShape().size() == 4) {
128-
cols = tensorInfo->getShape()[2];
129-
rows = tensorInfo->getShape()[1];
130-
} else if (tensorInfo->isInfluencedByDemultiplexer() && tensorInfo->getShape().size() == 5) {
131-
cols = tensorInfo->getShape()[3];
132-
rows = tensorInfo->getShape()[2];
133-
} else {
134-
return StatusCode::UNSUPPORTED_LAYOUT;
135-
}
136-
if (cols.isAny()) {
137-
cols = src.cols;
138-
}
139-
if (rows.isAny()) {
140-
rows = src.rows;
141-
}
142-
if (cols.isDynamic()) {
143-
dimension_value_t value = src.cols;
144-
if (src.cols < cols.getMinValue())
145-
value = cols.getMinValue();
146-
147-
if (src.cols > cols.getMaxValue())
148-
value = cols.getMaxValue();
149-
150-
if (value != src.cols)
151-
cols = Dimension(value);
152-
}
153-
if (rows.isDynamic()) {
154-
dimension_value_t value = src.rows;
155-
if (src.rows < rows.getMinValue())
156-
value = rows.getMinValue();
157-
158-
if (src.rows > rows.getMaxValue())
159-
value = rows.getMaxValue();
160-
161-
if (value != src.rows)
162-
rows = Dimension(value);
163-
}
164-
cv::resize(src, dst, cv::Size(cols.getStaticValue(), rows.getStaticValue()));
109+
Status resizeMat(const cv::Mat& src, cv::Mat& dst, const dimension_value_t height, const dimension_value_t width) {
110+
cv::resize(src, dst, cv::Size(width, height));
165111
return StatusCode::OK;
166112
}
167113

@@ -199,7 +145,7 @@ Status validateResolutionAgainstFirstBatchImage(const cv::Mat input, cv::Mat* fi
199145
if (input.cols == firstBatchImage->cols && input.rows == firstBatchImage->rows) {
200146
return StatusCode::OK;
201147
}
202-
SPDLOG_ERROR("Each binary image in request needs to have resolution matched. First cols: {}, rows: {}, current cols: {}, rows: {}",
148+
SPDLOG_DEBUG("Each binary image in request needs to have resolution matched. First cols: {}, rows: {}, current cols: {}, rows: {}",
203149
firstBatchImage->cols, firstBatchImage->rows, input.cols, input.rows);
204150
return StatusCode::BINARY_IMAGES_RESOLUTION_MISMATCH;
205151
}
@@ -212,14 +158,13 @@ bool checkBatchSizeMismatch(const std::shared_ptr<TensorInfo>& tensorInfo,
212158
return !tensorInfo->getBatchSize().value().match(batchSize);
213159
}
214160

215-
Status validateInput(const std::shared_ptr<TensorInfo>& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage) {
216-
// For pipelines with only custom nodes entry, or models with default layout there is no way to deduce layout.
217-
// With unknown layout, there is no way to deduce pipeline input resolution.
218-
// This forces binary utility to create tensors with resolution inherited from input binary image from request.
219-
// To achieve it, in this specific case we require all binary images to have the same resolution.
220-
// TODO check if H/W is undefined and only then check this CVS-77193
221-
if (firstBatchImage &&
222-
(tensorInfo->getLayout() == Layout::getUnspecifiedLayout())) {
161+
Status validateInput(const std::shared_ptr<TensorInfo>& tensorInfo, const cv::Mat input, cv::Mat* firstBatchImage, bool enforceResolutionAlignment) {
162+
// Binary inputs are supported for any endpoint that is compatible with N...HWC layout.
163+
// With unknown layout, there is no way to deduce expected endpoint input resolution.
164+
// This forces binary utility to create tensors with resolution inherited from first batch of binary input image (request).
165+
// In case of any dimension in endpoint shape is dynamic, we need to validate images against first image resolution.
166+
// Otherwise we can omit that, and proceed to image resize.
167+
if (firstBatchImage && enforceResolutionAlignment) {
223168
auto status = validateResolutionAgainstFirstBatchImage(input, firstBatchImage);
224169
if (!status.ok()) {
225170
return status;
@@ -258,17 +203,90 @@ Status validateTensor(const std::shared_ptr<TensorInfo>& tensorInfo,
258203
return StatusCode::OK;
259204
}
260205

206+
Dimension getTensorInfoHeightDim(const std::shared_ptr<TensorInfo>& tensorInfo) {
207+
size_t numberOfShapeDimensions = tensorInfo->getShape().size();
208+
if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) {
209+
throw std::logic_error("wrong number of shape dimensions");
210+
}
211+
size_t position = numberOfShapeDimensions == 4 ? /*NHWC*/ 1 : /*N?HWC*/ 2;
212+
return tensorInfo->getShape()[position];
213+
}
214+
215+
Dimension getTensorInfoWidthDim(const std::shared_ptr<TensorInfo>& tensorInfo) {
216+
size_t numberOfShapeDimensions = tensorInfo->getShape().size();
217+
if (numberOfShapeDimensions < 4 || numberOfShapeDimensions > 5) {
218+
throw std::logic_error("wrong number of shape dimensions");
219+
}
220+
size_t position = numberOfShapeDimensions == 4 ? /*NHWC*/ 2 : /*N?HWC*/ 3;
221+
return tensorInfo->getShape()[position];
222+
}
223+
224+
void updateTargetResolution(Dimension& height, Dimension& width, const cv::Mat& image) {
225+
if (height.isAny()) {
226+
height = image.rows;
227+
} else if (height.isDynamic()) {
228+
if (height.match(image.rows)) {
229+
height = image.rows;
230+
} else {
231+
if (image.rows > height.getMaxValue()) {
232+
height = height.getMaxValue();
233+
} else {
234+
height = height.getMinValue();
235+
}
236+
}
237+
}
238+
if (width.isAny()) {
239+
width = image.cols;
240+
} else if (width.isDynamic()) {
241+
if (width.match(image.cols)) {
242+
width = image.cols;
243+
} else {
244+
if (image.cols > width.getMaxValue()) {
245+
width = width.getMaxValue();
246+
} else {
247+
width = width.getMinValue();
248+
}
249+
}
250+
}
251+
}
252+
253+
bool isResizeSupported(const std::shared_ptr<TensorInfo>& tensorInfo) {
254+
for (const auto& dim : tensorInfo->getShape()) {
255+
if (dim.isAny()) {
256+
return false;
257+
}
258+
}
259+
if (tensorInfo->getLayout() != "NHWC" &&
260+
tensorInfo->getLayout() != "N?HWC" &&
261+
tensorInfo->getLayout() != Layout::getUnspecifiedLayout()) {
262+
return false;
263+
}
264+
return true;
265+
}
266+
261267
Status convertTensorToMatsMatchingTensorInfo(const tensorflow::TensorProto& src, std::vector<cv::Mat>& images, const std::shared_ptr<TensorInfo>& tensorInfo) {
268+
Dimension targetHeight = getTensorInfoHeightDim(tensorInfo);
269+
Dimension targetWidth = getTensorInfoWidthDim(tensorInfo);
270+
271+
// Enforce resolution alignment against first image in the batch if resize is not supported.
272+
bool resizeSupported = isResizeSupported(tensorInfo);
273+
bool enforceResolutionAlignment = !resizeSupported;
274+
262275
for (int i = 0; i < src.string_val_size(); i++) {
263276
cv::Mat image = convertStringValToMat(src.string_val(i));
264277
if (image.data == nullptr)
265278
return StatusCode::IMAGE_PARSING_FAILED;
266279

267280
cv::Mat* firstImage = images.size() == 0 ? nullptr : &images.at(0);
268-
auto status = validateInput(tensorInfo, image, firstImage);
281+
auto status = validateInput(tensorInfo, image, firstImage, enforceResolutionAlignment);
269282
if (status != StatusCode::OK) {
270283
return status;
271284
}
285+
286+
if (i == 0) {
287+
updateTargetResolution(targetHeight, targetWidth, image);
288+
}
289+
272290
if (!isPrecisionEqual(image.depth(), tensorInfo->getPrecision())) {
273291
cv::Mat imageCorrectPrecision;
274292
status = convertPrecision(image, imageCorrectPrecision, tensorInfo->getPrecision());
@@ -278,14 +296,26 @@ Status convertTensorToMatsMatchingTensorInfo(const tensorflow::TensorProto& src,
278296
}
279297
image = std::move(imageCorrectPrecision);
280298
}
281-
if (resizeNeeded(image, tensorInfo)) {
299+
if (!targetHeight.isStatic() || !targetWidth.isStatic()) {
300+
return StatusCode::INTERNAL_ERROR;
301+
}
302+
if (resizeNeeded(image, targetHeight.getStaticValue(), targetWidth.getStaticValue())) {
303+
if (!resizeSupported) {
304+
return StatusCode::INVALID_SHAPE;
305+
}
282306
cv::Mat imageResized;
283-
status = resizeMat(image, imageResized, tensorInfo);
307+
status = resizeMat(image, imageResized, targetHeight.getStaticValue(), targetWidth.getStaticValue());
284308
if (!status.ok()) {
285309
return status;
286310
}
287311
image = std::move(imageResized);
288312
}
313+
314+
if (i == 0 && src.string_val_size() > 1) {
315+
// TODO: CVS-78796 Check if the total bytes for tensor will not exceed 1GB.
316+
// Multiply src.string_val_size() * image resolution * precision size
317+
}
318+
289319
images.push_back(image);
290320
}
291321

@@ -304,7 +334,7 @@ shape_t getShapeFromImages(const std::vector<cv::Mat>& images, const std::shared
304334
return dims;
305335
}
306336

307-
ov::Tensor createTensorFromMats(const std::vector<cv::Mat>& images, const std::shared_ptr<TensorInfo>& tensorInfo, bool isPipeline) {
337+
ov::Tensor createTensorFromMats(const std::vector<cv::Mat>& images, const std::shared_ptr<TensorInfo>& tensorInfo) {
308338
ov::Shape shape = getShapeFromImages(images, tensorInfo);
309339
ov::element::Type precision = tensorInfo->getOvPrecision();
310340
ov::Tensor tensor(precision, shape);
@@ -316,7 +346,7 @@ ov::Tensor createTensorFromMats(const std::vector<cv::Mat>& images, const std::s
316346
return tensor;
317347
}
318348

319-
ov::Tensor convertMatsToTensor(std::vector<cv::Mat>& images, const std::shared_ptr<TensorInfo>& tensorInfo, bool isPipeline) {
349+
ov::Tensor convertMatsToTensor(std::vector<cv::Mat>& images, const std::shared_ptr<TensorInfo>& tensorInfo) {
320350
switch (tensorInfo->getPrecision()) {
321351
case ovms::Precision::FP32:
322352
case ovms::Precision::I32:
@@ -326,7 +356,7 @@ ov::Tensor convertMatsToTensor(std::vector<cv::Mat>& images, const std::shared_p
326356
case ovms::Precision::FP16:
327357
case ovms::Precision::U16:
328358
case ovms::Precision::I16:
329-
return createTensorFromMats(images, tensorInfo, isPipeline);
359+
return createTensorFromMats(images, tensorInfo);
330360
case ovms::Precision::MIXED:
331361
case ovms::Precision::Q78:
332362
case ovms::Precision::BIN:
@@ -337,7 +367,7 @@ ov::Tensor convertMatsToTensor(std::vector<cv::Mat>& images, const std::shared_p
337367
}
338368
}
339369

340-
Status convertStringValToTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr<TensorInfo>& tensorInfo, bool isPipeline) {
370+
Status convertStringValToTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr<TensorInfo>& tensorInfo) {
341371
auto status = validateTensor(tensorInfo, src);
342372
if (status != StatusCode::OK) {
343373
return status;
@@ -350,7 +380,7 @@ Status convertStringValToTensor(const tensorflow::TensorProto& src, ov::Tensor&
350380
return status;
351381
}
352382

353-
tensor = convertMatsToTensor(images, tensorInfo, isPipeline);
383+
tensor = convertMatsToTensor(images, tensorInfo);
354384
if (!tensor) {
355385
return StatusCode::IMAGE_PARSING_FAILED;
356386
}

‎src/binaryutils.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,5 +21,5 @@
2121
#include "tensorinfo.hpp"
2222

2323
namespace ovms {
24-
Status convertStringValToTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr<TensorInfo>& tensorInfo, bool isPipeline);
24+
Status convertStringValToTensor(const tensorflow::TensorProto& src, ov::Tensor& tensor, const std::shared_ptr<TensorInfo>& tensorInfo);
2525
} // namespace ovms

‎src/deserialization.hpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ Status deserializePredictRequest(
124124

125125
if (requestInput.dtype() == tensorflow::DataType::DT_STRING) {
126126
SPDLOG_DEBUG("Request contains binary input: {}", name);
127-
status = convertStringValToTensor(requestInput, tensor, tensorInfo, isPipeline);
127+
status = convertStringValToTensor(requestInput, tensor, tensorInfo);
128128
if (!status.ok()) {
129129
SPDLOG_DEBUG("Binary inputs conversion failed.");
130130
return status;

0 commit comments

Comments
 (0)