From 22652b576c2a0670d341648c68ca469ebe08f1a1 Mon Sep 17 00:00:00 2001 From: Wenbin Chen Date: Tue, 12 Dec 2023 10:33:31 +0800 Subject: [PATCH] libavfiter/dnn_backend_openvino: Add multiple output support Add multiple output support to openvino backend. You can use '&' to split different output when you set output name using command line. Signed-off-by: Wenbin Chen Reviewed-by: Guo Yejun --- libavfilter/dnn/dnn_backend_common.c | 7 - libavfilter/dnn/dnn_backend_openvino.c | 216 +++++++++++++++++-------- libavfilter/vf_dnn_detect.c | 11 +- 3 files changed, 150 insertions(+), 84 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_common.c b/libavfilter/dnn/dnn_backend_common.c index 91a4a3c4bf..632832ec36 100644 --- a/libavfilter/dnn/dnn_backend_common.c +++ b/libavfilter/dnn/dnn_backend_common.c @@ -43,13 +43,6 @@ int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func return AVERROR(EINVAL); } - if (exec_params->nb_output != 1 && backend != DNN_TF) { - // currently, the filter does not need multiple outputs, - // so we just pending the support until we really need it. - avpriv_report_missing_feature(ctx, "multiple outputs"); - return AVERROR(ENOSYS); - } - return 0; } diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c index 6fe8b9c243..089e028818 100644 --- a/libavfilter/dnn/dnn_backend_openvino.c +++ b/libavfilter/dnn/dnn_backend_openvino.c @@ -64,7 +64,7 @@ typedef struct OVModel{ ov_compiled_model_t *compiled_model; ov_output_const_port_t* input_port; ov_preprocess_input_info_t* input_info; - ov_output_const_port_t* output_port; + ov_output_const_port_t** output_ports; ov_preprocess_output_info_t* output_info; ov_preprocess_prepostprocessor_t* preprocess; #else @@ -77,6 +77,7 @@ typedef struct OVModel{ SafeQueue *request_queue; // holds OVRequestItem Queue *task_queue; // holds TaskItem Queue *lltask_queue; // holds LastLevelTaskItem + int nb_outputs; } OVModel; // one request for one call to openvino @@ -349,7 +350,7 @@ static void infer_completion_callback(void *args) TaskItem *task = lltask->task; OVModel *ov_model = task->model; SafeQueue *requestq = ov_model->request_queue; - DNNData output; + DNNData *outputs; OVContext *ctx = &ov_model->ctx; #if HAVE_OPENVINO2 size_t* dims; @@ -358,45 +359,61 @@ static void infer_completion_callback(void *args) ov_shape_t output_shape = {0}; ov_element_type_e precision; - memset(&output, 0, sizeof(output)); - status = ov_infer_request_get_output_tensor_by_index(request->infer_request, 0, &output_tensor); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, - "Failed to get output tensor."); + outputs = av_calloc(ov_model->nb_outputs, sizeof(*outputs)); + if (!outputs) { + av_log(ctx, AV_LOG_ERROR, "Failed to alloc outputs."); return; } - status = ov_tensor_data(output_tensor, &output.data); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, - "Failed to get output data."); - return; - } + for (int i = 0; i < ov_model->nb_outputs; i++) { + status = ov_infer_request_get_tensor_by_const_port(request->infer_request, + ov_model->output_ports[i], + &output_tensor); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, + "Failed to get output tensor."); + goto end; + } - status = ov_tensor_get_shape(output_tensor, &output_shape); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n"); - return; - } - dims = output_shape.dims; + status = ov_tensor_data(output_tensor, &outputs[i].data); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, + "Failed to get output data."); + goto end; + } - status = ov_port_get_element_type(ov_model->output_port, &precision); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n"); + status = ov_tensor_get_shape(output_tensor, &output_shape); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get output port shape.\n"); + goto end; + } + dims = output_shape.dims; + + status = ov_port_get_element_type(ov_model->output_ports[i], &precision); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get output port data type.\n"); + goto end; + } + outputs[i].dt = precision_to_datatype(precision); + + outputs[i].channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1; + outputs[i].height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1; + outputs[i].width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1; + av_assert0(request->lltask_count <= dims[0]); + outputs[i].layout = ctx->options.layout; + outputs[i].scale = ctx->options.scale; + outputs[i].mean = ctx->options.mean; ov_shape_free(&output_shape); - return; + ov_tensor_free(output_tensor); + output_tensor = NULL; } - output.channels = output_shape.rank > 2 ? dims[output_shape.rank - 3] : 1; - output.height = output_shape.rank > 1 ? dims[output_shape.rank - 2] : 1; - output.width = output_shape.rank > 0 ? dims[output_shape.rank - 1] : 1; - av_assert0(request->lltask_count <= dims[0]); - ov_shape_free(&output_shape); #else IEStatusCode status; dimensions_t dims; ie_blob_t *output_blob = NULL; ie_blob_buffer_t blob_buffer; precision_e precision; + DNNData output; status = ie_infer_request_get_blob(request->infer_request, task->output_names[0], &output_blob); if (status != OK) { av_log(ctx, AV_LOG_ERROR, @@ -424,11 +441,12 @@ static void infer_completion_callback(void *args) output.height = dims.dims[2]; output.width = dims.dims[3]; av_assert0(request->lltask_count <= dims.dims[0]); -#endif output.dt = precision_to_datatype(precision); output.layout = ctx->options.layout; output.scale = ctx->options.scale; output.mean = ctx->options.mean; + outputs = &output; +#endif av_assert0(request->lltask_count >= 1); for (int i = 0; i < request->lltask_count; ++i) { @@ -438,28 +456,33 @@ static void infer_completion_callback(void *args) case DFT_PROCESS_FRAME: if (task->do_ioproc) { if (ov_model->model->frame_post_proc != NULL) { - ov_model->model->frame_post_proc(task->out_frame, &output, ov_model->model->filter_ctx); + ov_model->model->frame_post_proc(task->out_frame, outputs, ov_model->model->filter_ctx); } else { - ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx); + ff_proc_from_dnn_to_frame(task->out_frame, outputs, ctx); } } else { - task->out_frame->width = output.width; - task->out_frame->height = output.height; + task->out_frame->width = outputs[0].width; + task->out_frame->height = outputs[0].height; } break; case DFT_ANALYTICS_DETECT: if (!ov_model->model->detect_post_proc) { av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n"); - return; + goto end; } - ov_model->model->detect_post_proc(task->in_frame, &output, 1, ov_model->model->filter_ctx); + ov_model->model->detect_post_proc(task->in_frame, outputs, + ov_model->nb_outputs, + ov_model->model->filter_ctx); break; case DFT_ANALYTICS_CLASSIFY: if (!ov_model->model->classify_post_proc) { av_log(ctx, AV_LOG_ERROR, "classify filter needs to provide post proc\n"); - return; + goto end; } - ov_model->model->classify_post_proc(task->in_frame, &output, request->lltasks[i]->bbox_index, ov_model->model->filter_ctx); + for (int output_i = 0; output_i < ov_model->nb_outputs; output_i++) + ov_model->model->classify_post_proc(task->in_frame, outputs, + request->lltasks[i]->bbox_index, + ov_model->model->filter_ctx); break; default: av_assert0(!"should not reach here"); @@ -468,10 +491,17 @@ static void infer_completion_callback(void *args) task->inference_done++; av_freep(&request->lltasks[i]); - output.data = (uint8_t *)output.data - + output.width * output.height * output.channels * get_datatype_size(output.dt); + for (int i = 0; i < ov_model->nb_outputs; i++) + outputs[i].data = (uint8_t *)outputs[i].data + + outputs[i].width * outputs[i].height * outputs[i].channels * get_datatype_size(outputs[i].dt); } -#if !HAVE_OPENVINO2 +end: +#if HAVE_OPENVINO2 + av_freep(&outputs); + ov_shape_free(&output_shape); + if (output_tensor) + ov_tensor_free(output_tensor); +#else ie_blob_free(&output_blob); #endif request->lltask_count = 0; @@ -525,8 +555,10 @@ static void dnn_free_model_ov(DNNModel **model) #if HAVE_OPENVINO2 if (ov_model->input_port) ov_output_const_port_free(ov_model->input_port); - if (ov_model->output_port) - ov_output_const_port_free(ov_model->output_port); + for (int i = 0; i < ov_model->nb_outputs; i++) + if (ov_model->output_ports[i]) + ov_output_const_port_free(ov_model->output_ports[i]); + av_freep(&ov_model->output_ports); if (ov_model->preprocess) ov_preprocess_prepostprocessor_free(ov_model->preprocess); if (ov_model->compiled_model) @@ -551,7 +583,7 @@ static void dnn_free_model_ov(DNNModel **model) } -static int init_model_ov(OVModel *ov_model, const char *input_name, const char *output_name) +static int init_model_ov(OVModel *ov_model, const char *input_name, const char **output_names, int nb_outputs) { int ret = 0; OVContext *ctx = &ov_model->ctx; @@ -594,17 +626,15 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * } status = ov_preprocess_prepostprocessor_get_input_info_by_name(ov_model->preprocess, input_name, &ov_model->input_info); - status |= ov_preprocess_prepostprocessor_get_output_info_by_name(ov_model->preprocess, output_name, &ov_model->output_info); if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get input/output info from preprocess.\n"); + av_log(ctx, AV_LOG_ERROR, "Failed to get input info from preprocess.\n"); ret = ov2_map_error(status, NULL); goto err; } status = ov_preprocess_input_info_get_tensor_info(ov_model->input_info, &input_tensor_info); - status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info); if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n"); + av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input.\n"); ret = ov2_map_error(status, NULL); goto err; } @@ -642,17 +672,43 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * } status = ov_preprocess_input_tensor_info_set_element_type(input_tensor_info, U8); - if (ov_model->model->func_type != DFT_PROCESS_FRAME) - status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); - else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) - status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); - else - status |= ov_preprocess_output_set_element_type(output_tensor_info, U8); if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to set input/output element type\n"); + av_log(ctx, AV_LOG_ERROR, "Failed to set input element type\n"); ret = ov2_map_error(status, NULL); goto err; } + + ov_model->nb_outputs = nb_outputs; + for (int i = 0; i < nb_outputs; i++) { + status = ov_preprocess_prepostprocessor_get_output_info_by_name( + ov_model->preprocess, output_names[i], &ov_model->output_info); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get output info from preprocess.\n"); + ret = ov2_map_error(status, NULL); + goto err; + } + status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n"); + ret = ov2_map_error(status, NULL); + goto err; + } + if (ov_model->model->func_type != DFT_PROCESS_FRAME) + status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); + else if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) + status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); + else + status |= ov_preprocess_output_set_element_type(output_tensor_info, U8); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to set output element type\n"); + ret = ov2_map_error(status, NULL); + goto err; + } + ov_preprocess_output_tensor_info_free(output_tensor_info); + output_tensor_info = NULL; + ov_preprocess_output_info_free(ov_model->output_info); + ov_model->output_info = NULL; + } // set preprocess steps. if (fabsf(ctx->options.scale - 1) > 1e-6f || fabsf(ctx->options.mean) > 1e-6f) { ov_preprocess_preprocess_steps_t* input_process_steps = NULL; @@ -667,11 +723,18 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * status |= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->options.scale); if (status != OK) { av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n"); + ov_preprocess_preprocess_steps_free(input_process_steps); + input_process_steps = NULL; ret = ov2_map_error(status, NULL); goto err; } ov_preprocess_preprocess_steps_free(input_process_steps); + input_process_steps = NULL; } + ov_preprocess_input_tensor_info_free(input_tensor_info); + input_tensor_info = NULL; + ov_preprocess_input_info_free(ov_model->input_info); + ov_model->input_info = NULL; //update model if(ov_model->ov_model) @@ -679,20 +742,33 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * status = ov_preprocess_prepostprocessor_build(ov_model->preprocess, &ov_model->ov_model); if (status != OK) { av_log(ctx, AV_LOG_ERROR, "Failed to update OV model\n"); + ov_model_free(tmp_ov_model); + tmp_ov_model = NULL; ret = ov2_map_error(status, NULL); goto err; } ov_model_free(tmp_ov_model); //update output_port - if (ov_model->output_port) { - ov_output_const_port_free(ov_model->output_port); - ov_model->output_port = NULL; - } - status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n"); - goto err; + if (!ov_model->output_ports) { + ov_model->output_ports = av_calloc(nb_outputs, sizeof(*ov_model->output_ports)); + if (!ov_model->output_ports) { + ret = AVERROR(ENOMEM); + goto err; + } + } else + for (int i = 0; i < nb_outputs; i++) { + ov_output_const_port_free(ov_model->output_ports[i]); + ov_model->output_ports[i] = NULL; + } + + for (int i = 0; i < nb_outputs; i++) { + status = ov_model_const_output_by_name(ov_model->ov_model, output_names[i], + &ov_model->output_ports[i]); + if (status != OK) { + av_log(ctx, AV_LOG_ERROR, "Failed to get output port %s.\n", output_names[i]); + goto err; + } } //compile network status = ov_core_compile_model(ov_model->core, ov_model->ov_model, device, 0, &ov_model->compiled_model); @@ -701,6 +777,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * goto err; } ov_preprocess_input_model_info_free(input_model_info); + input_model_info = NULL; ov_layout_free(NCHW_layout); ov_layout_free(NHWC_layout); #else @@ -745,6 +822,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * ret = DNN_GENERIC_ERROR; goto err; } + ov_model->nb_outputs = 1; // all models in openvino open model zoo use BGR with range [0.0f, 255.0f] as input, // we don't have a AVPixelFormat to describe it, so we'll use AV_PIX_FMT_BGR24 and @@ -848,6 +926,10 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * err: #if HAVE_OPENVINO2 + if (output_tensor_info) + ov_preprocess_output_tensor_info_free(output_tensor_info); + if (ov_model->output_info) + ov_preprocess_output_info_free(ov_model->output_info); if (NCHW_layout) ov_layout_free(NCHW_layout); if (NHWC_layout) @@ -1204,11 +1286,6 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i } } - status = ov_model_const_output_by_name(ov_model->ov_model, output_name, &ov_model->output_port); - if (status != OK) { - av_log(ctx, AV_LOG_ERROR, "Failed to get output port.\n"); - return ov2_map_error(status, NULL); - } if (!ov_model->compiled_model) { #else if (ctx->options.input_resizable) { @@ -1224,7 +1301,7 @@ static int get_output_ov(void *model, const char *input_name, int input_width, i } if (!ov_model->exe_network) { #endif - ret = init_model_ov(ov_model, input_name, output_name); + ret = init_model_ov(ov_model, input_name, &output_name, 1); if (ret != 0) { av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n"); return ret; @@ -1397,7 +1474,8 @@ static int dnn_execute_model_ov(const DNNModel *model, DNNExecBaseParams *exec_p #else if (!ov_model->exe_network) { #endif - ret = init_model_ov(ov_model, exec_params->input_name, exec_params->output_names[0]); + ret = init_model_ov(ov_model, exec_params->input_name, + exec_params->output_names, exec_params->nb_output); if (ret != 0) { av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n"); return ret; diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c index 3464af86c8..5862100b86 100644 --- a/libavfilter/vf_dnn_detect.c +++ b/libavfilter/vf_dnn_detect.c @@ -360,11 +360,11 @@ static int dnn_detect_post_proc_ssd(AVFrame *frame, DNNData *output, AVFilterCon break; } } - return 0; } -static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, AVFilterContext *filter_ctx) +static int dnn_detect_post_proc_ov(AVFrame *frame, DNNData *output, int nb_outputs, + AVFilterContext *filter_ctx) { AVFrameSideData *sd; DnnDetectContext *ctx = filter_ctx->priv; @@ -472,7 +472,7 @@ static int dnn_detect_post_proc(AVFrame *frame, DNNData *output, uint32_t nb, AV DnnContext *dnn_ctx = &ctx->dnnctx; switch (dnn_ctx->backend_type) { case DNN_OV: - return dnn_detect_post_proc_ov(frame, output, filter_ctx); + return dnn_detect_post_proc_ov(frame, output, nb, filter_ctx); case DNN_TF: return dnn_detect_post_proc_tf(frame, output, filter_ctx); default: @@ -559,11 +559,6 @@ static int check_output_nb(DnnDetectContext *ctx, DNNBackendType backend_type, i } return 0; case DNN_OV: - if (output_nb != 1) { - av_log(ctx, AV_LOG_ERROR, "Dnn detect filter with openvino backend needs 1 output only, \ - but get %d instead\n", output_nb); - return AVERROR(EINVAL); - } return 0; default: avpriv_report_missing_feature(ctx, "Dnn detect filter does not support current backend\n");