From 6aa7e07e7caed7997e40cee8b203ec56b12d7300 Mon Sep 17 00:00:00 2001 From: "Guo, Yejun" Date: Fri, 10 Apr 2020 21:35:11 +0800 Subject: [PATCH] dnn/native: add native support for 'add' It can be tested with the model file generated with below python script: import tensorflow as tf import numpy as np import imageio in_img = imageio.imread('input.jpg') in_img = in_img.astype(np.float32)/255.0 in_data = in_img[np.newaxis, :] x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name='dnn_in') z1 = 0.039 + x z2 = x + 0.042 z3 = z1 + z2 z4 = z3 - 0.381 z5 = z4 - x y = tf.math.maximum(z5, 0.0, name='dnn_out') sess=tf.Session() sess.run(tf.global_variables_initializer()) graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, ['dnn_out']) tf.train.write_graph(graph_def, '.', 'image_process.pb', as_text=False) print("image_process.pb generated, please use \ path_to_ffmpeg/tools/python/convert.py to generate image_process.model\n") output = sess.run(y, feed_dict={x: in_data}) imageio.imsave("out.jpg", np.squeeze(output)) Signed-off-by: Guo, Yejun --- .../dnn/dnn_backend_native_layer_mathbinary.c | 13 +++++++++++++ .../dnn/dnn_backend_native_layer_mathbinary.h | 1 + tools/python/convert_from_tensorflow.py | 15 +++++++-------- tools/python/convert_header.py | 2 +- 4 files changed, 22 insertions(+), 9 deletions(-) diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c index 3b8bab82bc..3fe337f730 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.c @@ -107,6 +107,19 @@ int dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_ope } } return 0; + case DMBO_ADD: + if (params->input0_broadcast || params->input1_broadcast) { + for (int i = 0; i < dims_count; ++i) { + dst[i] = params->v + src[i]; + } + } else { + const DnnOperand *input1 = &operands[input_operand_indexes[1]]; + const float *src1 = input1->data; + for (int i = 0; i < dims_count; ++i) { + dst[i] = src[i] + src1[i]; + } + } + return 0; default: return -1; } diff --git a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h index 6b684d1165..3c5bc6b2e1 100644 --- a/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h +++ b/libavfilter/dnn/dnn_backend_native_layer_mathbinary.h @@ -32,6 +32,7 @@ typedef enum { DMBO_SUB = 0, + DMBO_ADD = 1, DMBO_COUNT } DNNMathBinaryOperation; diff --git a/tools/python/convert_from_tensorflow.py b/tools/python/convert_from_tensorflow.py index 2485f16cd6..9a495c0a9e 100644 --- a/tools/python/convert_from_tensorflow.py +++ b/tools/python/convert_from_tensorflow.py @@ -71,7 +71,7 @@ class TFConverter: self.conv2d_scope_names = set() self.conv2d_scopename_inputname_dict = {} self.op2code = {'Conv2D':1, 'DepthToSpace':2, 'MirrorPad':3, 'Maximum':4, 'MathBinary':5} - self.mathbin2code = {'Sub':0} + self.mathbin2code = {'Sub':0, 'Add':1} self.mirrorpad_mode = {'CONSTANT':0, 'REFLECT':1, 'SYMMETRIC':2} self.name_operand_dict = {} @@ -255,8 +255,7 @@ class TFConverter: np.array([input_operand_index, output_operand_index], dtype=np.uint32).tofile(f) - def dump_sub_to_file(self, node, f): - assert(node.op == 'Sub') + def dump_mathbinary_to_file(self, node, f): self.layer_number = self.layer_number + 1 self.converted_nodes.add(node.name) i0_node = self.name_node_dict[node.input[0]] @@ -264,15 +263,13 @@ class TFConverter: np.array([self.op2code['MathBinary'], self.mathbin2code[node.op]], dtype=np.uint32).tofile(f) if i0_node.op == 'Const': scalar = i0_node.attr['value'].tensor.float_val[0] - assert(i0_node.name.find('sub/x')) - np.array([1], dtype=np.uint32).tofile(f) + np.array([1], dtype=np.uint32).tofile(f) # broadcast: 1 np.array([scalar], dtype=np.float32).tofile(f) - np.array([0], dtype=np.uint32).tofile(f) + np.array([0], dtype=np.uint32).tofile(f) # broadcast: 0 input_operand_index = self.add_operand(i1_node.name, Operand.IOTYPE_INPUT) np.array([input_operand_index], dtype=np.uint32).tofile(f) elif i1_node.op == 'Const': scalar = i1_node.attr['value'].tensor.float_val[0] - assert(i1_node.name.find('sub/y')) np.array([0], dtype=np.uint32).tofile(f) input_operand_index = self.add_operand(i0_node.name, Operand.IOTYPE_INPUT) np.array([input_operand_index], dtype=np.uint32).tofile(f) @@ -309,7 +306,9 @@ class TFConverter: elif node.op == 'Maximum': self.dump_maximum_to_file(node, f) elif node.op == 'Sub': - self.dump_sub_to_file(node, f) + self.dump_mathbinary_to_file(node, f) + elif node.op == 'Add': + self.dump_mathbinary_to_file(node, f) def dump_operands_to_file(self, f): diff --git a/tools/python/convert_header.py b/tools/python/convert_header.py index 6576fca7a1..70270225f1 100644 --- a/tools/python/convert_header.py +++ b/tools/python/convert_header.py @@ -23,4 +23,4 @@ str = 'FFMPEGDNNNATIVE' major = 1 # increase minor when we don't have to re-convert the model file -minor = 1 +minor = 2