2018-05-25 17:31:04 +00:00
|
|
|
/*
|
|
|
|
* Copyright (c) 2018 Sergey Lavrushkin
|
|
|
|
*
|
|
|
|
* This file is part of FFmpeg.
|
|
|
|
*
|
|
|
|
* FFmpeg is free software; you can redistribute it and/or
|
|
|
|
* modify it under the terms of the GNU Lesser General Public
|
|
|
|
* License as published by the Free Software Foundation; either
|
|
|
|
* version 2.1 of the License, or (at your option) any later version.
|
|
|
|
*
|
|
|
|
* FFmpeg is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
* Lesser General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU Lesser General Public
|
|
|
|
* License along with FFmpeg; if not, write to the Free Software
|
|
|
|
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
|
|
|
*/
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @file
|
|
|
|
* DNN inference functions interface for native backend.
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
2019-07-16 05:55:45 +00:00
|
|
|
#ifndef AVFILTER_DNN_DNN_BACKEND_NATIVE_H
|
|
|
|
#define AVFILTER_DNN_DNN_BACKEND_NATIVE_H
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2019-07-16 05:55:45 +00:00
|
|
|
#include "../dnn_interface.h"
|
2018-09-06 11:33:06 +00:00
|
|
|
#include "libavformat/avio.h"
|
2020-09-06 12:28:51 +00:00
|
|
|
#include "libavutil/opt.h"
|
2021-08-25 21:08:06 +00:00
|
|
|
#include "queue.h"
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2019-10-09 14:08:04 +00:00
|
|
|
/**
|
|
|
|
* the enum value of DNNLayerType should not be changed,
|
|
|
|
* the same values are used in convert_from_tensorflow.py
|
2019-10-09 14:08:18 +00:00
|
|
|
* and, it is used to index the layer execution/load function pointer.
|
2019-10-09 14:08:04 +00:00
|
|
|
*/
|
|
|
|
typedef enum {
|
|
|
|
DLT_INPUT = 0,
|
|
|
|
DLT_CONV2D = 1,
|
|
|
|
DLT_DEPTH_TO_SPACE = 2,
|
|
|
|
DLT_MIRROR_PAD = 3,
|
2019-10-09 14:08:11 +00:00
|
|
|
DLT_MAXIMUM = 4,
|
2020-03-20 12:55:38 +00:00
|
|
|
DLT_MATH_BINARY = 5,
|
dnn_backend_native_layer_mathunary: add abs support
more math unary operations will be added here
It can be tested with the model file generated with below python scripy:
import tensorflow as tf
import numpy as np
import imageio
in_img = imageio.imread('input.jpeg')
in_img = in_img.astype(np.float32)/255.0
in_data = in_img[np.newaxis, :]
x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in')
x1 = tf.subtract(x, 0.5)
x2 = tf.abs(x1)
y = tf.identity(x2, name='dnn_out')
sess=tf.Session()
sess.run(tf.global_variables_initializer())
graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out'])
tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False)
print("image_process.pb generated, please use \
path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n")
output = sess.run(y, feed_dict={x: in_data})
imageio.imsave("out.jpg", np.squeeze(output))
Signed-off-by: Ting Fu <ting.fu@intel.com>
Signed-off-by: Guo, Yejun <yejun.guo@intel.com>
2020-05-25 14:46:26 +00:00
|
|
|
DLT_MATH_UNARY = 6,
|
2020-08-09 16:33:13 +00:00
|
|
|
DLT_AVG_POOL = 7,
|
2020-09-22 07:11:09 +00:00
|
|
|
DLT_DENSE = 8,
|
2019-10-09 14:08:11 +00:00
|
|
|
DLT_COUNT
|
2019-10-09 14:08:04 +00:00
|
|
|
} DNNLayerType;
|
2018-09-06 11:33:06 +00:00
|
|
|
|
2020-06-01 01:00:45 +00:00
|
|
|
typedef enum {DOT_INPUT = 1, DOT_OUTPUT = 2, DOT_INTERMEDIATE = DOT_INPUT | DOT_OUTPUT} DNNOperandType;
|
2020-08-09 16:33:13 +00:00
|
|
|
typedef enum {VALID, SAME, SAME_CLAMP_TO_EDGE} DNNPaddingParam;
|
2020-09-22 07:11:09 +00:00
|
|
|
typedef enum {RELU, TANH, SIGMOID, NONE, LEAKY_RELU} DNNActivationFunc;
|
2019-08-29 05:53:33 +00:00
|
|
|
|
2018-09-06 11:33:06 +00:00
|
|
|
typedef struct Layer{
|
|
|
|
DNNLayerType type;
|
2019-08-29 05:53:33 +00:00
|
|
|
/**
|
|
|
|
* a layer can have multiple inputs and one output.
|
|
|
|
* 4 is just a big enough number for input operands (increase it if necessary),
|
|
|
|
* do not use 'int32_t *input_operand_indexes', so we don't worry about mem leaks.
|
|
|
|
*/
|
|
|
|
int32_t input_operand_indexes[4];
|
|
|
|
int32_t output_operand_index;
|
2018-09-06 11:33:06 +00:00
|
|
|
void *params;
|
|
|
|
} Layer;
|
|
|
|
|
2019-08-29 05:53:33 +00:00
|
|
|
typedef struct DnnOperand{
|
|
|
|
/**
|
|
|
|
* there are two memory layouts, NHWC or NCHW, so we use dims,
|
|
|
|
* dims[0] is Number.
|
|
|
|
*/
|
|
|
|
int32_t dims[4];
|
|
|
|
|
|
|
|
/**
|
|
|
|
* input/output/intermediate operand of the network
|
|
|
|
*/
|
|
|
|
DNNOperandType type;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* support different kinds of data type such as float, half float, int8 etc,
|
|
|
|
* first support float now.
|
|
|
|
*/
|
|
|
|
DNNDataType data_type;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* NHWC if 1, otherwise NCHW.
|
|
|
|
* let's first support NHWC only, this flag is for extensive usage.
|
|
|
|
*/
|
|
|
|
int8_t isNHWC;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* to avoid possible memory leak, do not use char *name
|
|
|
|
*/
|
|
|
|
char name[128];
|
|
|
|
|
|
|
|
/**
|
|
|
|
* data pointer with data length in bytes.
|
|
|
|
* usedNumbersLeft is only valid for intermediate operand,
|
|
|
|
* it means how many layers still depend on this operand,
|
|
|
|
* todo: the memory can be reused when usedNumbersLeft is zero.
|
|
|
|
*/
|
|
|
|
void *data;
|
|
|
|
int32_t length;
|
|
|
|
int32_t usedNumbersLeft;
|
|
|
|
}DnnOperand;
|
|
|
|
|
2018-09-06 11:33:06 +00:00
|
|
|
typedef struct InputParams{
|
|
|
|
int height, width, channels;
|
|
|
|
} InputParams;
|
|
|
|
|
2020-09-06 12:28:51 +00:00
|
|
|
typedef struct NativeOptions{
|
2021-08-25 21:10:45 +00:00
|
|
|
uint8_t async;
|
2020-09-06 12:28:51 +00:00
|
|
|
uint32_t conv2d_threads;
|
|
|
|
} NativeOptions;
|
|
|
|
|
2020-08-25 03:47:50 +00:00
|
|
|
typedef struct NativeContext {
|
|
|
|
const AVClass *class;
|
2020-09-06 12:28:51 +00:00
|
|
|
NativeOptions options;
|
2020-08-25 03:47:50 +00:00
|
|
|
} NativeContext;
|
|
|
|
|
2018-09-06 11:33:06 +00:00
|
|
|
// Represents simple feed-forward convolutional network.
|
2020-08-19 13:43:13 +00:00
|
|
|
typedef struct NativeModel{
|
2020-08-25 03:47:50 +00:00
|
|
|
NativeContext ctx;
|
2020-08-28 04:51:44 +00:00
|
|
|
DNNModel *model;
|
2018-09-06 11:33:06 +00:00
|
|
|
Layer *layers;
|
|
|
|
int32_t layers_num;
|
2019-08-29 05:53:33 +00:00
|
|
|
DnnOperand *operands;
|
|
|
|
int32_t operands_num;
|
2021-08-25 21:10:45 +00:00
|
|
|
Queue *task_queue;
|
2021-08-25 21:10:48 +00:00
|
|
|
Queue *lltask_queue;
|
2020-08-19 13:43:13 +00:00
|
|
|
} NativeModel;
|
2018-09-06 11:33:06 +00:00
|
|
|
|
2021-02-07 06:35:22 +00:00
|
|
|
DNNModel *ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx);
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
int ff_dnn_execute_model_native(const DNNModel *model, DNNExecBaseParams *exec_params);
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2021-08-25 21:10:45 +00:00
|
|
|
DNNAsyncStatusType ff_dnn_get_result_native(const DNNModel *model, AVFrame **in, AVFrame **out);
|
|
|
|
|
2022-03-02 18:05:54 +00:00
|
|
|
int ff_dnn_flush_native(const DNNModel *model);
|
2021-08-25 21:10:45 +00:00
|
|
|
|
2018-07-27 16:34:02 +00:00
|
|
|
void ff_dnn_free_model_native(DNNModel **model);
|
2018-05-25 17:31:04 +00:00
|
|
|
|
2020-07-06 07:32:17 +00:00
|
|
|
// NOTE: User must check for error (return value <= 0) to handle
|
|
|
|
// case like integer overflow.
|
2021-01-21 21:39:55 +00:00
|
|
|
int32_t ff_calculate_operand_data_length(const DnnOperand *oprd);
|
|
|
|
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd);
|
2018-05-25 17:31:04 +00:00
|
|
|
#endif
|