1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
// Copyright ManoMotion AB 2023
#include "OVBaseInference.hpp"
using namespace std;
using namespace manomotion;
using namespace cv;
OVBaseInference::OVBaseInference() {
TAG = "OVBaseInference";
is_read_from_file = false;
_batch_size = 1;
}
void OVBaseInference::normalizeImage(Mat image_frame, bool zero_center, float* tensor_buffer, bool pytorch_format) {
const int height = image_frame.rows;
const int width = image_frame.cols;
float div, sub;
if (zero_center) {
// [-1,1]
div = 127.5f;
sub = 1.0f;
}
else {
// [0,1]
div = 255.0f;
sub = 0.0f;
}
if (!pytorch_format) {
for (int i = 0; i < height; ++i) {
uchar* image_ptr = image_frame.data + i * width * channels;
for (int j = 0; j < width; ++j) {
for (int c = 0; c < channels; ++c) {
*tensor_buffer++ = (float)(*image_ptr++) / div - sub;
}
}
}
}
else {
for (int c = 0; c < channels; ++c) {
for (int i = 0; i < height; ++i) {
for (int j = 0; j < width; ++j) {
uchar* image_ptr = image_frame.data + i * width * channels + j * channels + c;
*tensor_buffer++ = (float)(*image_ptr) / div - sub;
}
}
}
}
return;
}
ov::InferRequest OVBaseInference::loadModel()
{
// Read a network in IR, PaddlePaddle, or ONNX format:
if (is_read_from_file)
{
try
{
model = core.read_model(model_name); }
catch (const std::exception& e)
{
cout << "Error while reading model : " << e.what() << endl;
}
}
else
{
//// read from embedded files
const char* embedded_file_name = model_name.c_str();
size_t file_size;
const char* _model_data;
if(is_multiple_embedded_files)
_model_data = find_embedded_file_multi(embedded_file_name, &file_size);
else
_model_data = find_embedded_file(embedded_file_name, &file_size);
// make string a from the read data
std::string model_str(_model_data, _model_data + file_size);
model = core.read_model(model_str, ov::Tensor());
}
model->reshape({ input_shape_new });
try
{
/*compiled_model = core.compile_model(model, "AUTO", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY),
// ov::inference_num_threads(8), ov::hint::inference_precision("f16"));*/
compiled_model = core.compile_model(model, "AUTO", ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY), ov::inference_num_threads(8));
//compiled_model = core.compile_model(model, "HETERO", ov::device::priorities("GPU,CPU"), ov::hint::performance_mode(ov::hint::PerformanceMode::LATENCY)); // crashes at infer
}
catch (const std::exception& e)
{
cout << "An exception occurred. Exception Nr. " << e.what() << '\n';
}
auto input_shape = model->input(input_info_idx).get_shape();
int size_full = 1;
for (int64_t _size : input_shape) {
dim.push_back(_size);
size_full *= _size;
}
input_name = *begin(model->input(input_info_idx).get_names());
output_name = *begin(model->output(output_info_idx).get_names());
assert(dim[2] == dim[3]); //ensure that we are processing a square inputs
input_dim = dim[2];
input_data = new float[size_full];
infer_request = compiled_model.create_infer_request();
return infer_request;
}
float* OVBaseInference::executeModel(int mode)
{
// Get input port for model with one input
auto input_port = compiled_model.input();
// Create tensor from external memory
ov::Tensor input_tensor(input_port.get_element_type(), input_port.get_shape(), input_data);
// Set input tensor for model with one input
infer_request.set_input_tensor(input_tensor);
switch (mode)
{
case 1:// sync
infer_request.infer();
break;
case 0:
default: //async
infer_request.start_async(); // there is a warning now"02 june 2023
infer_request.wait();
break;
}
// Get output tensor by tensor name
auto output = infer_request.get_tensor(output_name);
float* output_data = output.data<float>();
return output_data;
}
OVBaseInference::~OVBaseInference() {
}