2018-05-25 17:31:04 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2018 Sergey Lavrushkin
|
|
|
|
*
|
|
|
|
* This file is part of FFmpeg.
|
|
|
|
*
|
|
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @file
|
|
|
|
* DNN native backend implementation.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include "dnn_backend_native.h"
|
2019-04-25 02:14:42 +00:00
|
|
|
#include "libavutil/avassert.h"
|
2019-09-05 06:00:28 +00:00
|
|
|
#include "dnn_backend_native_layer_conv2d.h"
|
2019-10-09 14:08:11 +00:00
|
|
|
#include "dnn_backend_native_layers.h"
|
2020-08-28 04:51:44 +00:00
|
|
|
#include "dnn_io_proc.h"
|
2021-04-01 02:06:06 +00:00
|
|
|
#include "dnn_backend_common.h"
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2020-09-06 12:28:51 +00:00
|
|
|
#define OFFSET(x) offsetof(NativeContext, x)
|
|
|
|
#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
|
|
|
|
static const AVOption dnn_native_options[] = {
|
|
|
|
{ "conv2d_threads", "threads num for conv2d layer", OFFSET(options.conv2d_threads), AV_OPT_TYPE_INT, { .i64 = 0 }, INT_MIN, INT_MAX, FLAGS },
|
2021-08-25 21:10:45 +00:00
|
|
|
{ "async", "use DNN async inference", OFFSET(options.async), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
|
2020-09-06 12:28:51 +00:00
|
|
|
{ NULL },
|
|
|
|
};
|
|
|
|
|
2021-01-21 21:39:53 +00:00
|
|
|
static const AVClass dnn_native_class = {
|
2020-08-25 03:47:50 +00:00
|
|
|
.class_name = "dnn_native",
|
|
|
|
.item_name = av_default_item_name,
|
2020-09-06 12:28:51 +00:00
|
|
|
.option = dnn_native_options,
|
2020-08-25 03:47:50 +00:00
|
|
|
.version = LIBAVUTIL_VERSION_INT,
|
|
|
|
.category = AV_CLASS_CATEGORY_FILTER,
|
|
|
|
};
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
static int execute_model_native(Queue *lltask_queue);
|
2021-08-25 21:08:06 +00:00
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
|
2021-08-25 21:08:06 +00:00
|
|
|
{
|
|
|
|
NativeModel *native_model = task->model;
|
|
|
|
NativeContext *ctx = &native_model->ctx;
|
2021-08-25 21:10:48 +00:00
|
|
|
LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));
|
2021-08-25 21:08:06 +00:00
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
if (!lltask) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
return AVERROR(ENOMEM);
|
2021-08-25 21:08:06 +00:00
|
|
|
}
|
|
|
|
task->inference_todo = 1;
|
|
|
|
task->inference_done = 0;
|
2021-08-25 21:10:48 +00:00
|
|
|
lltask->task = task;
|
2021-08-25 21:08:06 +00:00
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
if (ff_queue_push_back(lltask_queue, lltask) < 0) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
|
|
|
|
av_freep(&lltask);
|
2022-03-02 18:05:54 +00:00
|
|
|
return AVERROR(ENOMEM);
|
2021-08-25 21:08:06 +00:00
|
|
|
}
|
2022-03-02 18:05:56 +00:00
|
|
|
return 0;
|
2021-08-25 21:08:06 +00:00
|
|
|
}
|
2020-09-11 14:15:04 +00:00
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
static int get_input_native(void *model, DNNData *input, const char *input_name)
|
2019-10-21 12:38:17 +00:00
|
|
|
{
|
2021-01-22 14:20:15 +00:00
|
|
|
NativeModel *native_model = model;
|
2020-08-25 03:47:50 +00:00
|
|
|
NativeContext *ctx = &native_model->ctx;
|
2019-10-21 12:38:17 +00:00
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
for (int i = 0; i < native_model->operands_num; ++i) {
|
|
|
|
DnnOperand *oprd = &native_model->operands[i];
|
2019-10-21 12:38:17 +00:00
|
|
|
if (strcmp(oprd->name, input_name) == 0) {
|
2020-08-25 03:47:50 +00:00
|
|
|
if (oprd->type != DOT_INPUT) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", input_name);
|
2022-03-02 18:05:54 +00:00
|
|
|
return AVERROR(EINVAL);
|
2020-08-25 03:47:50 +00:00
|
|
|
}
|
2019-10-21 12:38:17 +00:00
|
|
|
input->dt = oprd->data_type;
|
|
|
|
av_assert0(oprd->dims[0] == 1);
|
|
|
|
input->height = oprd->dims[1];
|
|
|
|
input->width = oprd->dims[2];
|
|
|
|
input->channels = oprd->dims[3];
|
2022-03-02 18:05:56 +00:00
|
|
|
return 0;
|
2019-10-21 12:38:17 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// do not find the input operand
|
2020-08-25 03:47:50 +00:00
|
|
|
av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
|
2022-03-02 18:05:54 +00:00
|
|
|
return AVERROR(EINVAL);
|
2019-10-21 12:38:17 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
static int get_output_native(void *model, const char *input_name, int input_width, int input_height,
|
2020-09-11 14:15:04 +00:00
|
|
|
const char *output_name, int *output_width, int *output_height)
|
|
|
|
{
|
2022-03-02 18:05:54 +00:00
|
|
|
int ret = 0;
|
2021-01-22 14:20:15 +00:00
|
|
|
NativeModel *native_model = model;
|
2020-10-14 00:19:50 +00:00
|
|
|
NativeContext *ctx = &native_model->ctx;
|
2021-08-25 21:08:06 +00:00
|
|
|
TaskItem task;
|
|
|
|
DNNExecBaseParams exec_params = {
|
|
|
|
.input_name = input_name,
|
|
|
|
.output_names = &output_name,
|
|
|
|
.nb_output = 1,
|
|
|
|
.in_frame = NULL,
|
|
|
|
.out_frame = NULL,
|
|
|
|
};
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, native_model, input_height, input_width, ctx);
|
2022-03-02 18:05:56 +00:00
|
|
|
if (ret != 0) {
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-10-14 00:19:50 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = extract_lltask_from_task(&task, native_model->lltask_queue);
|
2022-03-02 18:05:56 +00:00
|
|
|
if (ret != 0) {
|
2021-08-25 21:10:48 +00:00
|
|
|
av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-10-14 00:19:50 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
ret = execute_model_native(native_model->lltask_queue);
|
2021-08-25 21:08:06 +00:00
|
|
|
*output_width = task.out_frame->width;
|
|
|
|
*output_height = task.out_frame->height;
|
2020-09-11 14:15:04 +00:00
|
|
|
|
2021-08-25 21:08:06 +00:00
|
|
|
err:
|
|
|
|
av_frame_free(&task.out_frame);
|
|
|
|
av_frame_free(&task.in_frame);
|
2020-09-11 14:15:04 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2018-05-25 17:31:04 +00:00
|
|
|
// Loads model and its parameters that are stored in a binary file with following structure:
|
2018-06-13 21:37:12 +00:00
|
|
|
// layers_num,layer_type,layer_parameterss,layer_type,layer_parameters...
|
|
|
|
// For CONV layer: activation_function, input_num, output_num, kernel_size, kernel, biases
|
|
|
|
// For DEPTH_TO_SPACE layer: block_size
|
2021-02-07 06:35:22 +00:00
|
|
|
DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
|
2018-05-25 17:31:04 +00:00
|
|
|
{
|
2021-03-10 01:34:14 +00:00
|
|
|
#define DNN_NATIVE_MAGIC "FFMPEGDNNNATIVE"
|
2018-07-27 16:34:02 +00:00
|
|
|
DNNModel *model = NULL;
|
2021-03-10 01:34:14 +00:00
|
|
|
// sizeof - 1 to skip the terminating '\0' which is not written in the file
|
|
|
|
char buf[sizeof(DNN_NATIVE_MAGIC) - 1];
|
2019-10-21 12:38:03 +00:00
|
|
|
int version, header_size, major_version_expected = 1;
|
2020-08-19 13:43:13 +00:00
|
|
|
NativeModel *native_model = NULL;
|
2018-07-27 16:34:02 +00:00
|
|
|
AVIOContext *model_file_context;
|
2019-10-09 14:08:18 +00:00
|
|
|
int file_size, dnn_size, parsed_size;
|
2018-05-25 17:31:04 +00:00
|
|
|
int32_t layer;
|
2018-09-06 11:33:06 +00:00
|
|
|
DNNLayerType layer_type;
|
2018-05-25 17:31:04 +00:00
|
|
|
|
|
|
|
if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
file_size = avio_size(model_file_context);
|
|
|
|
|
2020-06-10 02:59:19 +00:00
|
|
|
model = av_mallocz(sizeof(DNNModel));
|
|
|
|
if (!model){
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
2019-09-02 04:35:58 +00:00
|
|
|
/**
|
|
|
|
* check file header with string and version
|
|
|
|
*/
|
2021-03-10 01:34:14 +00:00
|
|
|
if (avio_read(model_file_context, buf, sizeof(buf)) != sizeof(buf) ||
|
|
|
|
memcmp(buf, DNN_NATIVE_MAGIC, sizeof(buf)))
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2021-03-10 01:34:14 +00:00
|
|
|
dnn_size = sizeof(buf);
|
2019-09-02 04:35:58 +00:00
|
|
|
|
|
|
|
version = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
if (version != major_version_expected) {
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2019-09-02 04:35:58 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// currently no need to check minor version
|
|
|
|
version = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
header_size = dnn_size;
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
native_model = av_mallocz(sizeof(NativeModel));
|
|
|
|
if (!native_model){
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
2021-03-10 01:59:47 +00:00
|
|
|
model->model = native_model;
|
2020-08-25 03:47:50 +00:00
|
|
|
|
|
|
|
native_model->ctx.class = &dnn_native_class;
|
2020-09-06 12:28:51 +00:00
|
|
|
model->options = options;
|
|
|
|
if (av_opt_set_from_string(&native_model->ctx, model->options, NULL, "=", "&") < 0)
|
|
|
|
goto fail;
|
2020-08-28 04:51:44 +00:00
|
|
|
native_model->model = model;
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2021-08-25 21:10:45 +00:00
|
|
|
if (native_model->ctx.options.async) {
|
|
|
|
av_log(&native_model->ctx, AV_LOG_WARNING, "Async not supported. Rolling back to sync\n");
|
|
|
|
native_model->ctx.options.async = 0;
|
|
|
|
}
|
|
|
|
|
2020-09-06 12:28:51 +00:00
|
|
|
#if !HAVE_PTHREAD_CANCEL
|
|
|
|
if (native_model->ctx.options.conv2d_threads > 1){
|
|
|
|
av_log(&native_model->ctx, AV_LOG_WARNING, "'conv2d_threads' option was set but it is not supported "
|
|
|
|
"on this build (pthread support is required)\n");
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2019-08-20 08:50:34 +00:00
|
|
|
avio_seek(model_file_context, file_size - 8, SEEK_SET);
|
2020-08-19 13:43:13 +00:00
|
|
|
native_model->layers_num = (int32_t)avio_rl32(model_file_context);
|
|
|
|
native_model->operands_num = (int32_t)avio_rl32(model_file_context);
|
2019-09-02 04:35:58 +00:00
|
|
|
dnn_size += 8;
|
|
|
|
avio_seek(model_file_context, header_size, SEEK_SET);
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
native_model->layers = av_mallocz(native_model->layers_num * sizeof(Layer));
|
|
|
|
if (!native_model->layers){
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
native_model->operands = av_mallocz(native_model->operands_num * sizeof(DnnOperand));
|
|
|
|
if (!native_model->operands){
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 21:10:45 +00:00
|
|
|
native_model->task_queue = ff_queue_create();
|
|
|
|
if (!native_model->task_queue) {
|
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
native_model->lltask_queue = ff_queue_create();
|
|
|
|
if (!native_model->lltask_queue) {
|
2021-08-25 21:08:06 +00:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
for (layer = 0; layer < native_model->layers_num; ++layer){
|
2018-06-13 21:37:12 +00:00
|
|
|
layer_type = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
2019-10-09 14:08:18 +00:00
|
|
|
|
|
|
|
if (layer_type >= DLT_COUNT) {
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2019-10-09 14:08:18 +00:00
|
|
|
}
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
native_model->layers[layer].type = layer_type;
|
2021-01-21 21:39:55 +00:00
|
|
|
parsed_size = ff_layer_funcs[layer_type].pf_load(&native_model->layers[layer], model_file_context, file_size, native_model->operands_num);
|
2019-10-09 14:08:18 +00:00
|
|
|
if (!parsed_size) {
|
2020-06-10 02:59:19 +00:00
|
|
|
goto fail;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
2019-10-09 14:08:18 +00:00
|
|
|
dnn_size += parsed_size;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
for (int32_t i = 0; i < native_model->operands_num; ++i){
|
2019-08-20 08:50:34 +00:00
|
|
|
DnnOperand *oprd;
|
|
|
|
int32_t name_len;
|
|
|
|
int32_t operand_index = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
if (operand_index >= native_model->operands_num) {
|
2020-06-10 05:36:11 +00:00
|
|
|
goto fail;
|
|
|
|
}
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
oprd = &native_model->operands[operand_index];
|
2019-08-20 08:50:34 +00:00
|
|
|
name_len = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
|
|
|
|
avio_get_str(model_file_context, name_len, oprd->name, sizeof(oprd->name));
|
|
|
|
dnn_size += name_len;
|
|
|
|
|
|
|
|
oprd->type = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
|
|
|
|
oprd->data_type = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
|
|
|
|
for (int32_t dim = 0; dim < 4; ++dim) {
|
|
|
|
oprd->dims[dim] = (int32_t)avio_rl32(model_file_context);
|
|
|
|
dnn_size += 4;
|
|
|
|
}
|
2021-03-10 13:00:26 +00:00
|
|
|
if (oprd->type == DOT_INPUT && oprd->dims[0] != 1)
|
|
|
|
goto fail;
|
2019-08-20 08:50:34 +00:00
|
|
|
|
|
|
|
oprd->isNHWC = 1;
|
|
|
|
}
|
|
|
|
|
2018-05-25 17:31:04 +00:00
|
|
|
avio_closep(&model_file_context);
|
|
|
|
|
|
|
|
if (dnn_size != file_size){
|
|
|
|
ff_dnn_free_model_native(&model);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2019-10-21 12:38:17 +00:00
|
|
|
model->get_input = &get_input_native;
|
2020-09-11 14:15:04 +00:00
|
|
|
model->get_output = &get_output_native;
|
2020-11-18 06:54:10 +00:00
|
|
|
model->filter_ctx = filter_ctx;
|
2021-02-07 06:35:22 +00:00
|
|
|
model->func_type = func_type;
|
2018-05-25 17:31:04 +00:00
|
|
|
|
|
|
|
return model;
|
2020-06-10 02:59:19 +00:00
|
|
|
|
|
|
|
fail:
|
|
|
|
ff_dnn_free_model_native(&model);
|
|
|
|
avio_closep(&model_file_context);
|
|
|
|
return NULL;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
static int execute_model_native(Queue *lltask_queue)
|
2018-05-25 17:31:04 +00:00
|
|
|
{
|
2021-08-25 21:08:06 +00:00
|
|
|
NativeModel *native_model = NULL;
|
|
|
|
NativeContext *ctx = NULL;
|
2018-05-25 17:31:04 +00:00
|
|
|
int32_t layer;
|
2020-09-10 14:29:57 +00:00
|
|
|
DNNData input, output;
|
|
|
|
DnnOperand *oprd = NULL;
|
2021-08-25 21:10:48 +00:00
|
|
|
LastLevelTaskItem *lltask = NULL;
|
2021-08-25 21:08:06 +00:00
|
|
|
TaskItem *task = NULL;
|
2022-03-02 18:05:54 +00:00
|
|
|
int ret = 0;
|
2021-08-25 21:08:06 +00:00
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
lltask = ff_queue_pop_front(lltask_queue);
|
|
|
|
if (!lltask) {
|
|
|
|
av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(EINVAL);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
|
|
|
}
|
2021-08-25 21:10:48 +00:00
|
|
|
task = lltask->task;
|
2021-08-25 21:08:06 +00:00
|
|
|
native_model = task->model;
|
|
|
|
ctx = &native_model->ctx;
|
2020-08-28 04:51:44 +00:00
|
|
|
|
2020-09-10 14:29:57 +00:00
|
|
|
if (native_model->layers_num <= 0 || native_model->operands_num <= 0) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "No operands or layers in model\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(EINVAL);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-08-28 04:51:44 +00:00
|
|
|
}
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2020-09-10 14:29:57 +00:00
|
|
|
for (int i = 0; i < native_model->operands_num; ++i) {
|
|
|
|
oprd = &native_model->operands[i];
|
2021-08-25 21:08:06 +00:00
|
|
|
if (strcmp(oprd->name, task->input_name) == 0) {
|
2020-09-10 14:29:57 +00:00
|
|
|
if (oprd->type != DOT_INPUT) {
|
2021-08-25 21:08:06 +00:00
|
|
|
av_log(ctx, AV_LOG_ERROR, "Found \"%s\" in model, but it is not input node\n", task->input_name);
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(EINVAL);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-09-10 14:29:57 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
oprd = NULL;
|
|
|
|
}
|
|
|
|
if (!oprd) {
|
2021-08-25 21:08:06 +00:00
|
|
|
av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(EINVAL);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-08-25 03:47:50 +00:00
|
|
|
}
|
2020-09-10 14:29:57 +00:00
|
|
|
|
2021-08-25 21:08:06 +00:00
|
|
|
oprd->dims[1] = task->in_frame->height;
|
|
|
|
oprd->dims[2] = task->in_frame->width;
|
2020-09-10 14:29:57 +00:00
|
|
|
|
|
|
|
av_freep(&oprd->data);
|
2021-01-21 21:39:55 +00:00
|
|
|
oprd->length = ff_calculate_operand_data_length(oprd);
|
2020-09-10 14:29:57 +00:00
|
|
|
if (oprd->length <= 0) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "The input data length overflow\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(EINVAL);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-09-10 14:29:57 +00:00
|
|
|
}
|
|
|
|
oprd->data = av_malloc(oprd->length);
|
|
|
|
if (!oprd->data) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "Failed to malloc memory for input data\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(ENOMEM);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-09-10 14:29:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
input.height = oprd->dims[1];
|
|
|
|
input.width = oprd->dims[2];
|
|
|
|
input.channels = oprd->dims[3];
|
|
|
|
input.data = oprd->data;
|
|
|
|
input.dt = oprd->data_type;
|
2021-08-25 21:08:06 +00:00
|
|
|
if (task->do_ioproc) {
|
2021-03-01 11:23:20 +00:00
|
|
|
if (native_model->model->frame_pre_proc != NULL) {
|
2021-08-25 21:08:06 +00:00
|
|
|
native_model->model->frame_pre_proc(task->in_frame, &input, native_model->model->filter_ctx);
|
2020-09-11 14:15:04 +00:00
|
|
|
} else {
|
2021-08-25 21:08:06 +00:00
|
|
|
ff_proc_from_frame_to_dnn(task->in_frame, &input, ctx);
|
2020-09-11 14:15:04 +00:00
|
|
|
}
|
2020-09-10 14:29:57 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 21:08:06 +00:00
|
|
|
if (task->nb_output != 1) {
|
2020-09-10 14:29:57 +00:00
|
|
|
// currently, the filter does not need multiple outputs,
|
|
|
|
// so we just pending the support until we really need it.
|
2021-01-18 07:56:34 +00:00
|
|
|
avpriv_report_missing_feature(ctx, "multiple outputs");
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(ENOSYS);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-08-25 03:47:50 +00:00
|
|
|
}
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
for (layer = 0; layer < native_model->layers_num; ++layer){
|
|
|
|
DNNLayerType layer_type = native_model->layers[layer].type;
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = ff_layer_funcs[layer_type].pf_exec(native_model->operands,
|
|
|
|
native_model->layers[layer].input_operand_indexes,
|
|
|
|
native_model->layers[layer].output_operand_index,
|
|
|
|
native_model->layers[layer].params,
|
|
|
|
&native_model->ctx);
|
2022-03-02 18:05:56 +00:00
|
|
|
if (ret != 0) {
|
2021-03-10 14:27:52 +00:00
|
|
|
av_log(ctx, AV_LOG_ERROR, "Failed to execute model\n");
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-08-25 03:47:49 +00:00
|
|
|
}
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 21:08:06 +00:00
|
|
|
for (uint32_t i = 0; i < task->nb_output; ++i) {
|
2020-08-13 08:19:48 +00:00
|
|
|
DnnOperand *oprd = NULL;
|
2021-08-25 21:08:06 +00:00
|
|
|
const char *output_name = task->output_names[i];
|
2020-08-13 08:19:48 +00:00
|
|
|
for (int j = 0; j < native_model->operands_num; ++j) {
|
|
|
|
if (strcmp(native_model->operands[j].name, output_name) == 0) {
|
|
|
|
oprd = &native_model->operands[j];
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-08-25 03:47:50 +00:00
|
|
|
if (oprd == NULL) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "Could not find output in model\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = AVERROR(EINVAL);
|
2021-08-25 21:08:06 +00:00
|
|
|
goto err;
|
2020-08-25 03:47:50 +00:00
|
|
|
}
|
2020-08-13 08:19:48 +00:00
|
|
|
|
2020-08-28 04:51:44 +00:00
|
|
|
output.data = oprd->data;
|
|
|
|
output.height = oprd->dims[1];
|
|
|
|
output.width = oprd->dims[2];
|
|
|
|
output.channels = oprd->dims[3];
|
|
|
|
output.dt = oprd->data_type;
|
|
|
|
|
2021-08-25 21:08:06 +00:00
|
|
|
if (task->do_ioproc) {
|
2021-03-01 11:23:20 +00:00
|
|
|
if (native_model->model->frame_post_proc != NULL) {
|
2021-08-25 21:08:06 +00:00
|
|
|
native_model->model->frame_post_proc(task->out_frame, &output, native_model->model->filter_ctx);
|
2020-08-28 04:51:44 +00:00
|
|
|
} else {
|
2021-08-25 21:08:06 +00:00
|
|
|
ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
|
2020-08-28 04:51:44 +00:00
|
|
|
}
|
2020-09-11 14:15:04 +00:00
|
|
|
} else {
|
2021-08-25 21:08:06 +00:00
|
|
|
task->out_frame->width = output.width;
|
|
|
|
task->out_frame->height = output.height;
|
2020-08-28 04:51:44 +00:00
|
|
|
}
|
2019-09-20 03:56:10 +00:00
|
|
|
}
|
2021-08-25 21:08:06 +00:00
|
|
|
task->inference_done++;
|
|
|
|
err:
|
2021-08-25 21:10:48 +00:00
|
|
|
av_freep(&lltask);
|
2021-08-25 21:08:06 +00:00
|
|
|
return ret;
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
int ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params)
|
2020-09-11 14:15:04 +00:00
|
|
|
{
|
2021-01-22 14:20:15 +00:00
|
|
|
NativeModel *native_model = model->model;
|
2020-09-11 14:15:04 +00:00
|
|
|
NativeContext *ctx = &native_model->ctx;
|
2021-08-25 21:10:45 +00:00
|
|
|
TaskItem *task;
|
2022-03-02 18:05:54 +00:00
|
|
|
int ret = 0;
|
2020-09-11 14:15:04 +00:00
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = ff_check_exec_params(ctx, DNN_NATIVE, model->func_type, exec_params);
|
|
|
|
if (ret != 0) {
|
|
|
|
return ret;
|
2020-09-11 14:15:04 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 21:10:45 +00:00
|
|
|
task = av_malloc(sizeof(*task));
|
|
|
|
if (!task) {
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
return AVERROR(ENOMEM);
|
2021-08-25 21:08:06 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = ff_dnn_fill_task(task, exec_params, native_model, ctx->options.async, 1);
|
2022-03-02 18:05:56 +00:00
|
|
|
if (ret != 0) {
|
2021-08-25 21:10:45 +00:00
|
|
|
av_freep(&task);
|
2022-03-02 18:05:54 +00:00
|
|
|
return ret;
|
2021-08-25 21:10:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (ff_queue_push_back(native_model->task_queue, task) < 0) {
|
|
|
|
av_freep(&task);
|
|
|
|
av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
return AVERROR(ENOMEM);
|
2021-08-25 21:10:45 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
ret = extract_lltask_from_task(task, native_model->lltask_queue);
|
2022-03-02 18:05:56 +00:00
|
|
|
if (ret != 0) {
|
2021-08-25 21:10:48 +00:00
|
|
|
av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
|
2022-03-02 18:05:54 +00:00
|
|
|
return ret;
|
2021-08-25 21:08:06 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
return execute_model_native(native_model->lltask_queue);
|
2020-09-11 14:15:04 +00:00
|
|
|
}
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
int ff_dnn_flush_native(const DNNModel *model)
|
2021-08-25 21:10:45 +00:00
|
|
|
{
|
|
|
|
NativeModel *native_model = model->model;
|
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
if (ff_queue_size(native_model->lltask_queue) == 0) {
|
2021-08-25 21:10:45 +00:00
|
|
|
// no pending task need to flush
|
2022-03-02 18:05:56 +00:00
|
|
|
return 0;
|
2021-08-25 21:10:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// for now, use sync node with flush operation
|
|
|
|
// Switch to async when it is supported
|
2021-08-25 21:10:48 +00:00
|
|
|
return execute_model_native(native_model->lltask_queue);
|
2021-08-25 21:10:45 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out)
|
|
|
|
{
|
|
|
|
NativeModel *native_model = model->model;
|
|
|
|
return ff_dnn_get_result_common(native_model->task_queue, in, out);
|
|
|
|
}
|
|
|
|
|
2021-01-21 21:39:55 +00:00
|
|
|
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
|
2019-09-20 03:55:48 +00:00
|
|
|
{
|
|
|
|
int32_t result = 1;
|
|
|
|
for (int i = 0; i < 4; ++i)
|
|
|
|
result *= oprd->dims[i];
|
|
|
|
|
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2021-01-21 21:39:55 +00:00
|
|
|
int32_t ff_calculate_operand_data_length(const DnnOperand* oprd)
|
2019-08-29 05:53:33 +00:00
|
|
|
{
|
|
|
|
// currently, we just support DNN_FLOAT
|
2020-07-06 07:32:17 +00:00
|
|
|
uint64_t len = sizeof(float);
|
|
|
|
for (int i = 0; i < 4; i++) {
|
|
|
|
len *= oprd->dims[i];
|
|
|
|
if (len > INT32_MAX)
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return len;
|
2019-08-29 05:53:33 +00:00
|
|
|
}
|
|
|
|
|
2018-07-27 16:34:02 +00:00
|
|
|
void ff_dnn_free_model_native(DNNModel **model)
|
2018-05-25 17:31:04 +00:00
|
|
|
{
|
2020-08-19 13:43:13 +00:00
|
|
|
NativeModel *native_model;
|
2018-07-27 16:34:02 +00:00
|
|
|
ConvolutionalParams *conv_params;
|
2018-05-25 17:31:04 +00:00
|
|
|
int32_t layer;
|
|
|
|
|
|
|
|
if (*model)
|
|
|
|
{
|
2020-06-10 02:59:19 +00:00
|
|
|
if ((*model)->model) {
|
2021-01-22 14:20:15 +00:00
|
|
|
native_model = (*model)->model;
|
2020-08-19 13:43:13 +00:00
|
|
|
if (native_model->layers) {
|
|
|
|
for (layer = 0; layer < native_model->layers_num; ++layer){
|
|
|
|
if (native_model->layers[layer].type == DLT_CONV2D){
|
|
|
|
conv_params = (ConvolutionalParams *)native_model->layers[layer].params;
|
2020-06-10 02:59:19 +00:00
|
|
|
av_freep(&conv_params->kernel);
|
|
|
|
av_freep(&conv_params->biases);
|
|
|
|
}
|
2020-08-19 13:43:13 +00:00
|
|
|
av_freep(&native_model->layers[layer].params);
|
2020-06-10 02:59:19 +00:00
|
|
|
}
|
2020-08-19 13:43:13 +00:00
|
|
|
av_freep(&native_model->layers);
|
2018-05-25 17:31:04 +00:00
|
|
|
}
|
2019-08-29 05:53:33 +00:00
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
if (native_model->operands) {
|
|
|
|
for (uint32_t operand = 0; operand < native_model->operands_num; ++operand)
|
|
|
|
av_freep(&native_model->operands[operand].data);
|
|
|
|
av_freep(&native_model->operands);
|
2020-06-10 02:59:19 +00:00
|
|
|
}
|
2019-08-29 05:53:33 +00:00
|
|
|
|
2021-08-25 21:10:48 +00:00
|
|
|
while (ff_queue_size(native_model->lltask_queue) != 0) {
|
|
|
|
LastLevelTaskItem *item = ff_queue_pop_front(native_model->lltask_queue);
|
2021-08-25 21:08:06 +00:00
|
|
|
av_freep(&item);
|
|
|
|
}
|
2021-08-25 21:10:48 +00:00
|
|
|
ff_queue_destroy(native_model->lltask_queue);
|
2021-08-25 21:10:45 +00:00
|
|
|
|
|
|
|
while (ff_queue_size(native_model->task_queue) != 0) {
|
|
|
|
TaskItem *item = ff_queue_pop_front(native_model->task_queue);
|
|
|
|
av_frame_free(&item->in_frame);
|
|
|
|
av_frame_free(&item->out_frame);
|
|
|
|
av_freep(&item);
|
|
|
|
}
|
|
|
|
ff_queue_destroy(native_model->task_queue);
|
|
|
|
|
2020-08-19 13:43:13 +00:00
|
|
|
av_freep(&native_model);
|
2020-06-10 02:59:19 +00:00
|
|
|
}
|
2018-05-25 17:31:04 +00:00
|
|
|
av_freep(model);
|
|
|
|
}
|
|
|
|
}
|