ffmpeg/libavfilter/dnn/dnn_backend_native_layer_ma...

157 lines
4.8 KiB
C
Raw Normal View History

/*
* Copyright (c) 2020
*
* This file is part of FFmpeg.
*
* FFmpeg is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* FFmpeg is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with FFmpeg; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
/**
* @file
* DNN native backend implementation.
*/
#include <math.h>
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_mathunary.h"
int ff_dnn_load_layer_math_unary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
{
DnnLayerMathUnaryParams *params;
int dnn_size = 0;
params = av_malloc(sizeof(*params));
if(!params)
return 0;
params->un_op = (int32_t)avio_rl32(model_file_context);
dnn_size += 4;
layer->params = params;
layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
dnn_size += 8;
if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
return 0;
}
return dnn_size;
}
int ff_dnn_execute_layer_math_unary(DnnOperand *operands, const int32_t *input_operand_indexes,
int32_t output_operand_index, const void *parameters, NativeContext *ctx)
{
const DnnOperand *input = &operands[input_operand_indexes[0]];
DnnOperand *output = &operands[output_operand_index];
const DnnLayerMathUnaryParams *params = parameters;
int dims_count;
const float *src;
float *dst;
for (int i = 0; i < 4; ++i)
output->dims[i] = input->dims[i];
output->data_type = input->data_type;
output->length = ff_calculate_operand_data_length(output);
if (output->length <= 0) {
av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
return AVERROR(EINVAL);
}
output->data = av_realloc(output->data, output->length);
if (!output->data) {
av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
return AVERROR(ENOMEM);
}
dims_count = ff_calculate_operand_dims_count(output);
src = input->data;
dst = output->data;
switch (params->un_op) {
case DMUO_ABS:
for (int i = 0; i < dims_count; ++i)
dst[i] = FFABS(src[i]);
return 0;
case DMUO_SIN:
for (int i = 0; i < dims_count; ++i)
dst[i] = sin(src[i]);
return 0;
case DMUO_COS:
for (int i = 0; i < dims_count; ++i)
dst[i] = cos(src[i]);
return 0;
case DMUO_TAN:
for (int i = 0; i < dims_count; ++i)
dst[i] = tan(src[i]);
return 0;
case DMUO_ASIN:
for (int i = 0; i < dims_count; ++i)
dst[i] = asin(src[i]);
return 0;
case DMUO_ACOS:
for (int i = 0; i < dims_count; ++i)
dst[i] = acos(src[i]);
return 0;
case DMUO_ATAN:
for (int i = 0; i < dims_count; ++i)
dst[i] = atan(src[i]);
return 0;
case DMUO_SINH:
for (int i = 0; i < dims_count; ++i)
dst[i] = sinh(src[i]);
return 0;
case DMUO_COSH:
for (int i = 0; i < dims_count; ++i)
dst[i] = cosh(src[i]);
return 0;
case DMUO_TANH:
for (int i = 0; i < dims_count; ++i)
dst[i] = tanh(src[i]);
return 0;
case DMUO_ASINH:
for (int i = 0; i < dims_count; ++i)
dst[i] = asinh(src[i]);
return 0;
case DMUO_ACOSH:
for (int i = 0; i < dims_count; ++i)
dst[i] = acosh(src[i]);
return 0;
dnn_backend_native_layer_mathunary: add atanh support It can be tested with the model generated with below python script: import tensorflow as tf import numpy as np import imageio in_img = imageio.imread('input.jpeg') in_img = in_img.astype(np.float32)/255.0 in_data = in_img[np.newaxis, :] x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in') please uncomment the part you want to test x_sinh_1 = tf.sinh(x) x_out = tf.divide(x_sinh_1, 1.176) # sinh(1.0) x_cosh_1 = tf.cosh(x) x_out = tf.divide(x_cosh_1, 1.55) # cosh(1.0) x_tanh_1 = tf.tanh(x) x__out = tf.divide(x_tanh_1, 0.77) # tanh(1.0) x_asinh_1 = tf.asinh(x) x_out = tf.divide(x_asinh_1, 0.89) # asinh(1.0/1.1) x_acosh_1 = tf.add(x, 1.1) x_acosh_2 = tf.acosh(x_acosh_1) # accept (1, inf) x_out = tf.divide(x_acosh_2, 1.4) # acosh(2.1) x_atanh_1 = tf.divide(x, 1.1) x_atanh_2 = tf.atanh(x_atanh_1) # accept (-1, 1) x_out = tf.divide(x_atanh_2, 1.55) # atanhh(1.0/1.1) y = tf.identity(x_out, name='dnn_out') #please only preserve the x_out you want to test sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False) print("image_process.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n") output = sess.run(y, feed_dict={x: in_data}) imageio.imsave("out.jpg", np.squeeze(output)) Signed-off-by: Ting Fu <ting.fu@intel.com>
2020-06-29 14:54:10 +00:00
case DMUO_ATANH:
for (int i = 0; i < dims_count; ++i)
dst[i] = atanh(src[i]);
return 0;
dnn_backend_native_layer_mathunary: add ceil support It can be tested with the model generated with below python script: import tensorflow as tf import os import numpy as np import imageio from tensorflow.python.framework import graph_util name = 'ceil' pb_file_path = os.getcwd() if not os.path.exists(pb_file_path+'/{}_savemodel/'.format(name)): os.mkdir(pb_file_path+'/{}_savemodel/'.format(name)) with tf.Session(graph=tf.Graph()) as sess: in_img = imageio.imread('detection.jpg') in_img = in_img.astype(np.float32) in_data = in_img[np.newaxis, :] input_x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in') y = tf.math.ceil( input_x, name='dnn_out') sess.run(tf.global_variables_initializer()) constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) with tf.gfile.FastGFile(pb_file_path+'/{}_savemodel/model.pb'.format(name), mode='wb') as f: f.write(constant_graph.SerializeToString()) print("model.pb generated, please in ffmpeg path use\n \n \ python tools/python/convert.py ceil_savemodel/model.pb --outdir=ceil_savemodel/ \n \n \ to generate model.model\n") output = sess.run(y, feed_dict={ input_x: in_data}) imageio.imsave("out.jpg", np.squeeze(output)) print("To verify, please ffmpeg path use\n \n \ ./ffmpeg -i detection.jpg -vf format=rgb24,dnn_processing=model=ceil_savemodel/model.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow -f framemd5 ceil_savemodel/tensorflow_out.md5\n \n \ to generate output result of tensorflow model\n") print("To verify, please ffmpeg path use\n \n \ ./ffmpeg -i detection.jpg -vf format=rgb24,dnn_processing=model=ceil_savemodel/model.model:input=dnn_in:output=dnn_out:dnn_backend=native -f framemd5 ceil_savemodel/native_out.md5\n \n \ to generate output result of native model\n") Signed-off-by: Mingyu Yin <mingyu.yin@intel.com> Reviewed-by: Guo, Yejun <yejun.guo@intel.com>
2020-07-31 07:41:24 +00:00
case DMUO_CEIL:
for (int i = 0; i < dims_count; ++i)
dst[i] = ceil(src[i]);
return 0;
dnn_backend_native_layer_mathunary: add floor support It can be tested with the model generated with below python script: import tensorflow as tf import os import numpy as np import imageio from tensorflow.python.framework import graph_util name = 'floor' pb_file_path = os.getcwd() if not os.path.exists(pb_file_path+'/{}_savemodel/'.format(name)): os.mkdir(pb_file_path+'/{}_savemodel/'.format(name)) with tf.Session(graph=tf.Graph()) as sess: in_img = imageio.imread('detection.jpg') in_img = in_img.astype(np.float32) in_data = in_img[np.newaxis, :] input_x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in') y_ = tf.math.floor(input_x*255)/255 y = tf.identity(y_, name='dnn_out') sess.run(tf.global_variables_initializer()) constant_graph = graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) with tf.gfile.FastGFile(pb_file_path+'/{}_savemodel/model.pb'.format(name), mode='wb') as f: f.write(constant_graph.SerializeToString()) print("model.pb generated, please in ffmpeg path use\n \n \ python tools/python/convert.py {}_savemodel/model.pb --outdir={}_savemodel/ \n \nto generate model.model\n".format(name,name)) output = sess.run(y, feed_dict={ input_x: in_data}) imageio.imsave("out.jpg", np.squeeze(output)) print("To verify, please ffmpeg path use\n \n \ ./ffmpeg -i detection.jpg -vf format=rgb24,dnn_processing=model={}_savemodel/model.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow -f framemd5 {}_savemodel/tensorflow_out.md5\n \ or\n \ ./ffmpeg -i detection.jpg -vf format=rgb24,dnn_processing=model={}_savemodel/model.pb:input=dnn_in:output=dnn_out:dnn_backend=tensorflow {}_savemodel/out_tensorflow.jpg\n \nto generate output result of tensorflow model\n".format(name, name, name, name)) print("To verify, please ffmpeg path use\n \n \ ./ffmpeg -i detection.jpg -vf format=rgb24,dnn_processing=model={}_savemodel/model.model:input=dnn_in:output=dnn_out:dnn_backend=native -f framemd5 {}_savemodel/native_out.md5\n \ or \n \ ./ffmpeg -i detection.jpg -vf format=rgb24,dnn_processing=model={}_savemodel/model.model:input=dnn_in:output=dnn_out:dnn_backend=native {}_savemodel/out_native.jpg\n \nto generate output result of native model\n".format(name, name, name, name)) Signed-off-by: Mingyu Yin <mingyu.yin@intel.com>
2020-08-06 06:47:16 +00:00
case DMUO_FLOOR:
for (int i = 0; i < dims_count; ++i)
dst[i] = floor(src[i]);
return 0;
case DMUO_ROUND:
for (int i = 0; i < dims_count; ++i)
dst[i] = round(src[i]);
return 0;
case DMUO_EXP:
for (int i = 0; i < dims_count; ++i)
dst[i] = exp(src[i]);
return 0;
default:
av_log(ctx, AV_LOG_ERROR, "Unmatch math unary operator\n");
return AVERROR(EINVAL);
}
}