From a761da9a093242e0a35fadaf3d58049555c9702c Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 11:40:26 +0100 Subject: [PATCH 01/14] Added support for `Resize` node from QONNX model --- hls4ml/backends/fpga/passes/clone.py | 2 +- hls4ml/converters/onnx/reshape.py | 20 ++++++- hls4ml/converters/onnx_to_hls.py | 6 +++ hls4ml/model/layers.py | 52 ++++++++++++++----- hls4ml/model/optimizer/__init__.py | 1 + hls4ml/model/optimizer/passes/resize_const.py | 25 +++++++++ 6 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 hls4ml/model/optimizer/passes/resize_const.py diff --git a/hls4ml/backends/fpga/passes/clone.py b/hls4ml/backends/fpga/passes/clone.py index 306e839900..0c1f7f2e07 100644 --- a/hls4ml/backends/fpga/passes/clone.py +++ b/hls4ml/backends/fpga/passes/clone.py @@ -87,7 +87,7 @@ def transform(self, model, node): ) for i in range(len(output_map[output])): key = output + '_cpy' + str(i + 1) - clone_layer.attributes[key].type = node.attributes['result_t'] + clone_layer.attributes[key].type = node.get_output_variable().type model.insert_node(clone_layer) transformed = True diff --git a/hls4ml/converters/onnx/reshape.py b/hls4ml/converters/onnx/reshape.py index 9ef20f03d7..09fb8a1510 100644 --- a/hls4ml/converters/onnx/reshape.py +++ b/hls4ml/converters/onnx/reshape.py @@ -1,4 +1,4 @@ -from hls4ml.converters.onnx_to_hls import onnx_handler +from hls4ml.converters.onnx_to_hls import get_onnx_attribute, onnx_handler @onnx_handler('Transpose') @@ -36,3 +36,21 @@ def parse_flatten_layer(node, input_names, input_shapes, graph): layer['target_shape'] = [-1] # does not contain batch dimension return layer + +@onnx_handler('Resize') +def parse_resize_layer(node, input_names, input_shapes, graph): + layer = {} + layer['name'] = node.name + layer['class_name'] = 'Resize' + layer['inputs'] = input_names + layer['outputs'] = list(node.output) + layer['in_height'] = input_shapes[0][2] + layer['in_width'] = input_shapes[0][1] + layer['out_width'] = input_shapes[0][1] + layer['out_height'] = input_shapes[0][2] + layer['n_chan'] = input_shapes[0][3] + layer['algorithm'] = get_onnx_attribute(node, 'mode') + # The followin is used in initialize() method. Probably a better solution would be to have a channels last parameter at QONNX level + layer['data_format'] = 'channels_last' if any(node.domain == 'qonnx.custom_op.channels_last' for node in graph.node) else 'channels_first' + + return layer \ No newline at end of file diff --git a/hls4ml/converters/onnx_to_hls.py b/hls4ml/converters/onnx_to_hls.py index 75850fa93e..eea6a261e8 100644 --- a/hls4ml/converters/onnx_to_hls.py +++ b/hls4ml/converters/onnx_to_hls.py @@ -63,6 +63,12 @@ def get_input_shape(graph, node): """ rv = [] for inp in node.input: + # this couple of lines doesn't look very nice but I don't think it would be considered as wrong. + # It is necessary for `Resize` node, since RoI input is empty but necessary to specify also scales + # array. It might be better handled in QONNX, refers to this issue for more details: + # https://github.com/fastmachinelearning/qonnx/issues/150 + if inp == '': + continue try: value_info_idx = next((i for i, x in enumerate(graph.value_info) if x.name == inp)) dim = list(d.dim_value for d in graph.value_info[value_info_idx].type.tensor_type.shape.dim) diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index fb548aa164..9bf6ea3b2d 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1143,20 +1143,46 @@ class Resize(Layer): def initialize(self): inp = self.get_input_variable() - if self.get_attr('data_format') == 'channels_last': - if len(inp.shape) == 2: # 1D -> width + chan - shape = [self.get_attr('out_width'), self.get_attr('n_chan')] - dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] - elif len(inp.shape) == 3: # 2D -> height + width + chan - shape = [self.get_attr('out_height'), self.get_attr('out_width'), self.get_attr('n_chan')] - dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] + if len(self.inputs) > 1: + # get the scales of Resize node from QONNX frontend + scales = self.get_input_node(self.inputs[-1]).get_attr('value') + if self.get_attr('data_format') == 'channels_last': + if len(inp.shape) == 2: # 1D -> width + chan + shape = [int(self.get_attr('out_width') * scales[1]), int(self.get_attr('n_chan') * scales[2])] + dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] + elif len(inp.shape) == 3: # 2D -> height + width + chan + shape = [ + int(self.get_attr('out_height') * scales[1]), + int(self.get_attr('out_width') * scales[2]), + int(self.get_attr('n_chan') * scales[3]), + ] + dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] + else: + if len(inp.shape) == 2: # 1D -> width + chan + shape = [int(self.get_attr('n_chan') * scales[1]), int(self.get_attr('out_width') * scales[2])] + dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}'] + elif len(inp.shape) == 3: # 2D -> height + width + chan + shape = [ + int(self.get_attr('n_chan') * scales[1]), + int(self.get_attr('out_height') * scales[2]), + int(self.get_attr('out_width') * scales[3]) + ] + dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}'] else: - if len(inp.shape) == 2: # 1D -> width + chan - shape = [self.get_attr('n_chan'), self.get_attr('out_width')] - dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}'] - elif len(inp.shape) == 3: # 2D -> height + width + chan - shape = [self.get_attr('n_chan'), self.get_attr('out_height'), self.get_attr('out_width')] - dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}'] + if self.get_attr('data_format') == 'channels_last': + if len(inp.shape) == 2: # 1D -> width + chan + shape = [self.get_attr('out_width'), self.get_attr('n_chan')] + dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] + elif len(inp.shape) == 3: # 2D -> height + width + chan + shape = [self.get_attr('out_height'), self.get_attr('out_width'), self.get_attr('n_chan')] + dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] + else: + if len(inp.shape) == 2: # 1D -> width + chan + shape = [self.get_attr('n_chan'), self.get_attr('out_width')] + dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}'] + elif len(inp.shape) == 3: # 2D -> height + width + chan + shape = [self.get_attr('n_chan'), self.get_attr('out_height'), self.get_attr('out_width')] + dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}'] self.add_output_variable(shape, dims, precision=inp.type.precision) diff --git a/hls4ml/model/optimizer/__init__.py b/hls4ml/model/optimizer/__init__.py index 0edd549b29..abd2af1101 100644 --- a/hls4ml/model/optimizer/__init__.py +++ b/hls4ml/model/optimizer/__init__.py @@ -34,6 +34,7 @@ 'parse_qonnx', [ 'reshape_constant', + 'resize_constant', 'quant_constant_parameters', 'quant_to_activation', 'fuse_quant_with_constant', diff --git a/hls4ml/model/optimizer/passes/resize_const.py b/hls4ml/model/optimizer/passes/resize_const.py new file mode 100644 index 0000000000..14547d1053 --- /dev/null +++ b/hls4ml/model/optimizer/passes/resize_const.py @@ -0,0 +1,25 @@ +from hls4ml.model.layers import Constant, Resize +from hls4ml.model.optimizer import OptimizerPass + +class ResizeConstant(OptimizerPass): + """ + To compute the output shape of resize is necessary to access the scales, that + are stored as initilizer, later on converted as constant inputs. + """ + def match(self, node): + is_match = isinstance(node, Resize) and len(node.inputs) > 1 and node.get_input_node(node.inputs[-1]) + return is_match + + def transform(self, model, node): + """ + Remove Constant from new shape input. Note, input shape node is already used on initialize + """ + scales_node = node.get_input_node(node.inputs[-1]) + node.inputs[-1] = '' + scales_values = scales_node.get_attr('value') + node.set_attr('out_width', int(node.get_attr('in_width') * scales_values[1])) + node.set_attr('out_height', int(node.get_attr('in_height') * scales_values[2])) + if not isinstance(scales_node, Constant): + raise RuntimeError("Non-constant shape inputs are not supported") + model.remove_node(scales_node, rewire=False) + return True \ No newline at end of file From b62468ac39a0a513326e388d90df648cfd64a9fb Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 14:01:42 +0100 Subject: [PATCH 02/14] Added a test on tiny UNet model in order to test `Resize` node --- test/pytest/test_qonnx.py | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/test/pytest/test_qonnx.py b/test/pytest/test_qonnx.py index f822c591a7..7448505a7d 100644 --- a/test/pytest/test_qonnx.py +++ b/test/pytest/test_qonnx.py @@ -12,6 +12,7 @@ from qonnx.core.modelwrapper import ModelWrapper from qonnx.transformation.channels_last import ConvertToChannelsLastAndClean from qonnx.transformation.gemm_to_matmul import GemmToMatMul +from qonnx.transformation.base import Transformation import hls4ml @@ -100,6 +101,18 @@ def sep_conv_model(): return model +@pytest.fixture(scope='module') +def tiny_unet_model(): + """ + Load tiny unet model, already channels-last and cleaned + """ + dl_file = str(example_model_path / "onnx/tiny_unet_ch_last.onnx") + assert os.path.isfile(dl_file) + + model = ModelWrapper(dl_file) + + return model + @pytest.fixture(scope='module') def two_layer_keras_model(): @@ -309,6 +322,50 @@ def test_sep_conv(sep_conv_model, backend): np.testing.assert_allclose(y_qonnx.ravel(), y_hls4ml.ravel(), atol=1e-2, rtol=1) +@pytest.mark.parametrize('backend', ['Vitis']) +def test_tiny_unet_model(tiny_unet_model, backend): + class EmptyFilledRoI(Transformation): + "Remove RoI tensor of Resize node added for shape inference" + + def apply(self, model): + graph_modified = False + for node in model.graph.node: + if node.op_type == 'Resize': + # Assuming 'roi' is the second input + if len(node.input) > 2 and node.input[1] != '': + init_names = [x.name for x in model.graph.initializer] + i = init_names.index(node.input[1]) + init_to_remove = model.graph.initializer[i] + model.graph.initializer.remove(init_to_remove) + node.input[1] = '' + graph_modified = True + return (model, graph_modified) + + model = tiny_unet_model + ishape = tuple(model.get_tensor_shape(model.graph.input[0].name)) + X = np.random.uniform(low=0, high=1, size=np.prod(ishape)).reshape(ishape) + X = (np.round(X * 2**16) * 2**-16).astype(np.float32) + idict = {model.graph.input[0].name: X} + y_qonnx = oxe.execute_onnx(model, idict)[model.graph.output[0].name] + + config = hls4ml.utils.config.config_from_onnx_model( + model, granularity='name', backend=backend, default_precision='fixed<32,16>' + ) + + model = model.transform(EmptyFilledRoI()) + hls_model = hls4ml.converters.convert_from_onnx_model( + model, + output_dir=str(test_root_path / f'hls4mlprj_qonnx_tiny_unet_model_{backend}'), + io_type='io_stream', + backend=backend, + hls_config=config, + ) + hls_model.compile() + y_hls4ml = hls_model.predict(np.ascontiguousarray(X)) + + np.testing.assert_array_equal(y_qonnx.ravel(), y_hls4ml.ravel(), atol=1e-2, rtol=1) + + @pytest.mark.parametrize( 'model_name', [ From 40a431fe20b29d2af8cf92fd9fda6ff6defca583 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 14:25:12 +0100 Subject: [PATCH 03/14] pre-commit restyling --- hls4ml/converters/onnx/reshape.py | 9 ++++++--- hls4ml/model/layers.py | 16 ++++++++-------- hls4ml/model/optimizer/passes/resize_const.py | 4 +++- test/pytest/test_qonnx.py | 5 +++-- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/hls4ml/converters/onnx/reshape.py b/hls4ml/converters/onnx/reshape.py index 09fb8a1510..212dd7b450 100644 --- a/hls4ml/converters/onnx/reshape.py +++ b/hls4ml/converters/onnx/reshape.py @@ -37,6 +37,7 @@ def parse_flatten_layer(node, input_names, input_shapes, graph): return layer + @onnx_handler('Resize') def parse_resize_layer(node, input_names, input_shapes, graph): layer = {} @@ -50,7 +51,9 @@ def parse_resize_layer(node, input_names, input_shapes, graph): layer['out_height'] = input_shapes[0][2] layer['n_chan'] = input_shapes[0][3] layer['algorithm'] = get_onnx_attribute(node, 'mode') - # The followin is used in initialize() method. Probably a better solution would be to have a channels last parameter at QONNX level - layer['data_format'] = 'channels_last' if any(node.domain == 'qonnx.custom_op.channels_last' for node in graph.node) else 'channels_first' + # The followin is used in initialize() method. Probably a better solution would be to have a channels last parameter at QONNX level + layer['data_format'] = ( + 'channels_last' if any(node.domain == 'qonnx.custom_op.channels_last' for node in graph.node) else 'channels_first' + ) - return layer \ No newline at end of file + return layer diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index 9bf6ea3b2d..f7cef37b10 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1144,7 +1144,7 @@ def initialize(self): inp = self.get_input_variable() if len(self.inputs) > 1: - # get the scales of Resize node from QONNX frontend + # get the scales of Resize node from QONNX frontend scales = self.get_input_node(self.inputs[-1]).get_attr('value') if self.get_attr('data_format') == 'channels_last': if len(inp.shape) == 2: # 1D -> width + chan @@ -1152,10 +1152,10 @@ def initialize(self): dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] elif len(inp.shape) == 3: # 2D -> height + width + chan shape = [ - int(self.get_attr('out_height') * scales[1]), - int(self.get_attr('out_width') * scales[2]), - int(self.get_attr('n_chan') * scales[3]), - ] + int(self.get_attr('out_height') * scales[1]), + int(self.get_attr('out_width') * scales[2]), + int(self.get_attr('n_chan') * scales[3]), + ] dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] else: if len(inp.shape) == 2: # 1D -> width + chan @@ -1163,9 +1163,9 @@ def initialize(self): dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}'] elif len(inp.shape) == 3: # 2D -> height + width + chan shape = [ - int(self.get_attr('n_chan') * scales[1]), - int(self.get_attr('out_height') * scales[2]), - int(self.get_attr('out_width') * scales[3]) + int(self.get_attr('n_chan') * scales[1]), + int(self.get_attr('out_height') * scales[2]), + int(self.get_attr('out_width') * scales[3]), ] dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}'] else: diff --git a/hls4ml/model/optimizer/passes/resize_const.py b/hls4ml/model/optimizer/passes/resize_const.py index 14547d1053..cd82d19d93 100644 --- a/hls4ml/model/optimizer/passes/resize_const.py +++ b/hls4ml/model/optimizer/passes/resize_const.py @@ -1,11 +1,13 @@ from hls4ml.model.layers import Constant, Resize from hls4ml.model.optimizer import OptimizerPass + class ResizeConstant(OptimizerPass): """ To compute the output shape of resize is necessary to access the scales, that are stored as initilizer, later on converted as constant inputs. """ + def match(self, node): is_match = isinstance(node, Resize) and len(node.inputs) > 1 and node.get_input_node(node.inputs[-1]) return is_match @@ -22,4 +24,4 @@ def transform(self, model, node): if not isinstance(scales_node, Constant): raise RuntimeError("Non-constant shape inputs are not supported") model.remove_node(scales_node, rewire=False) - return True \ No newline at end of file + return True diff --git a/test/pytest/test_qonnx.py b/test/pytest/test_qonnx.py index 7448505a7d..1002b6c146 100644 --- a/test/pytest/test_qonnx.py +++ b/test/pytest/test_qonnx.py @@ -10,9 +10,9 @@ # To conveniently run QONNX inference from qonnx.core.modelwrapper import ModelWrapper +from qonnx.transformation.base import Transformation from qonnx.transformation.channels_last import ConvertToChannelsLastAndClean from qonnx.transformation.gemm_to_matmul import GemmToMatMul -from qonnx.transformation.base import Transformation import hls4ml @@ -101,6 +101,7 @@ def sep_conv_model(): return model + @pytest.fixture(scope='module') def tiny_unet_model(): """ @@ -331,7 +332,7 @@ def apply(self, model): graph_modified = False for node in model.graph.node: if node.op_type == 'Resize': - # Assuming 'roi' is the second input + # Assuming 'roi' is the second input if len(node.input) > 2 and node.input[1] != '': init_names = [x.name for x in model.graph.initializer] i = init_names.index(node.input[1]) From be55945f23ee6041bd24c08960ce9aaad8f6d441 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 14:32:43 +0100 Subject: [PATCH 04/14] Aesthetic fix --- hls4ml/converters/onnx/reshape.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hls4ml/converters/onnx/reshape.py b/hls4ml/converters/onnx/reshape.py index 212dd7b450..e7b3129be5 100644 --- a/hls4ml/converters/onnx/reshape.py +++ b/hls4ml/converters/onnx/reshape.py @@ -51,7 +51,8 @@ def parse_resize_layer(node, input_names, input_shapes, graph): layer['out_height'] = input_shapes[0][2] layer['n_chan'] = input_shapes[0][3] layer['algorithm'] = get_onnx_attribute(node, 'mode') - # The followin is used in initialize() method. Probably a better solution would be to have a channels last parameter at QONNX level + # The following is used in initialize() method. + # Probably a better solution would be to have a channels last parameter at QONNX level layer['data_format'] = ( 'channels_last' if any(node.domain == 'qonnx.custom_op.channels_last' for node in graph.node) else 'channels_first' ) From 743831f22fc74d43d983d706fad5a31c2be700ea Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 14:38:39 +0100 Subject: [PATCH 05/14] Second aesthetic fix --- hls4ml/converters/onnx/reshape.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hls4ml/converters/onnx/reshape.py b/hls4ml/converters/onnx/reshape.py index e7b3129be5..f11796b6db 100644 --- a/hls4ml/converters/onnx/reshape.py +++ b/hls4ml/converters/onnx/reshape.py @@ -51,7 +51,7 @@ def parse_resize_layer(node, input_names, input_shapes, graph): layer['out_height'] = input_shapes[0][2] layer['n_chan'] = input_shapes[0][3] layer['algorithm'] = get_onnx_attribute(node, 'mode') - # The following is used in initialize() method. + # The following is used in initialize() method. # Probably a better solution would be to have a channels last parameter at QONNX level layer['data_format'] = ( 'channels_last' if any(node.domain == 'qonnx.custom_op.channels_last' for node in graph.node) else 'channels_first' From 4f82810bb5b3afcc4f72d8eca9aa2644e91784fd Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 16:08:41 +0100 Subject: [PATCH 06/14] Added one test on a simpler model extracted from UNet model `branched_model_ch_last.onnx` --- test/pytest/test_qonnx.py | 76 ++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/test/pytest/test_qonnx.py b/test/pytest/test_qonnx.py index 1002b6c146..75dd80dcdc 100644 --- a/test/pytest/test_qonnx.py +++ b/test/pytest/test_qonnx.py @@ -102,6 +102,19 @@ def sep_conv_model(): return model +@pytest.fixture(scope='module') +def branched_model(): + """ + Load branched model using separable convs, already channels-last and cleaned + """ + dl_file = str(example_model_path / "onnx/branched_model_ch_last.onnx") + assert os.path.isfile(dl_file) + + model = ModelWrapper(dl_file) + + return model + + @pytest.fixture(scope='module') def tiny_unet_model(): """ @@ -323,24 +336,53 @@ def test_sep_conv(sep_conv_model, backend): np.testing.assert_allclose(y_qonnx.ravel(), y_hls4ml.ravel(), atol=1e-2, rtol=1) +class EmptyFilledRoI(Transformation): + "Remove RoI tensor of Resize node added for shape inference" + + def apply(self, model): + graph_modified = False + for node in model.graph.node: + if node.op_type == 'Resize': + # Assuming 'roi' is the second input + if len(node.input) > 2 and node.input[1] != '': + init_names = [x.name for x in model.graph.initializer] + i = init_names.index(node.input[1]) + init_to_remove = model.graph.initializer[i] + model.graph.initializer.remove(init_to_remove) + node.input[1] = '' + graph_modified = True + return (model, graph_modified) + + +@pytest.mark.parametrize('backend', ['Vitis']) +def test_branched_model(branched_model, backend): + model = branched_model + ishape = tuple(model.get_tensor_shape(model.graph.input[0].name)) + X = np.random.uniform(low=0, high=1, size=np.prod(ishape)).reshape(ishape) + X = (np.round(X * 2**16) * 2**-16).astype(np.float32) + idict = {model.graph.input[0].name: X} + y_qonnx = oxe.execute_onnx(model, idict)[model.graph.output[0].name] + + config = hls4ml.utils.config.config_from_onnx_model( + model, granularity='name', backend=backend, default_precision='fixed<32,16>' + ) + + model = model.transform(EmptyFilledRoI()) + hls_model = hls4ml.converters.convert_from_onnx_model( + model, + output_dir=str(test_root_path / f'hls4mlprj_qonnx_branched_model_{backend}'), + io_type='io_stream', + backend=backend, + hls_config=config, + ) + hls_model.compile() + y_hls4ml = hls_model.predict(np.ascontiguousarray(X)) + + np.testing.assert_array_equal(y_qonnx.ravel(), y_hls4ml.ravel()) + + @pytest.mark.parametrize('backend', ['Vitis']) def test_tiny_unet_model(tiny_unet_model, backend): - class EmptyFilledRoI(Transformation): - "Remove RoI tensor of Resize node added for shape inference" - - def apply(self, model): - graph_modified = False - for node in model.graph.node: - if node.op_type == 'Resize': - # Assuming 'roi' is the second input - if len(node.input) > 2 and node.input[1] != '': - init_names = [x.name for x in model.graph.initializer] - i = init_names.index(node.input[1]) - init_to_remove = model.graph.initializer[i] - model.graph.initializer.remove(init_to_remove) - node.input[1] = '' - graph_modified = True - return (model, graph_modified) model = tiny_unet_model ishape = tuple(model.get_tensor_shape(model.graph.input[0].name)) @@ -364,7 +406,7 @@ def apply(self, model): hls_model.compile() y_hls4ml = hls_model.predict(np.ascontiguousarray(X)) - np.testing.assert_array_equal(y_qonnx.ravel(), y_hls4ml.ravel(), atol=1e-2, rtol=1) + np.testing.assert_array_equal(y_qonnx.ravel(), y_hls4ml.ravel()) @pytest.mark.parametrize( From 7e6b9af7db9d51b736d65fb694be6344297a7a4f Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 22:30:28 +0100 Subject: [PATCH 07/14] Example models commit updated --- example-models | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example-models b/example-models index d40894b03f..6a82da23ad 160000 --- a/example-models +++ b/example-models @@ -1 +1 @@ -Subproject commit d40894b03f840a32da43a5adea0531ffc1db216e +Subproject commit 6a82da23ad24c238fe156ed4d0aa907db547dbcf From 5757ac62bd54885ac491ce3ba06a86a985dbc1ca Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 12 Nov 2024 23:19:39 +0100 Subject: [PATCH 08/14] An empty list is now appended to the shape of all the inputs of the considered node, in case the input is empty --- hls4ml/converters/onnx_to_hls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hls4ml/converters/onnx_to_hls.py b/hls4ml/converters/onnx_to_hls.py index eea6a261e8..6410434625 100644 --- a/hls4ml/converters/onnx_to_hls.py +++ b/hls4ml/converters/onnx_to_hls.py @@ -68,6 +68,7 @@ def get_input_shape(graph, node): # array. It might be better handled in QONNX, refers to this issue for more details: # https://github.com/fastmachinelearning/qonnx/issues/150 if inp == '': + rv.append([]) continue try: value_info_idx = next((i for i, x in enumerate(graph.value_info) if x.name == inp)) From cf80f64cb42a265be46368811a7a07e8658a2cd9 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Sat, 16 Nov 2024 00:12:59 +0100 Subject: [PATCH 09/14] Cleaned some code and added the removal of RoI input from `Resize` node --- hls4ml/model/layers.py | 29 +++++++++---- hls4ml/model/optimizer/__init__.py | 2 +- hls4ml/model/optimizer/passes/linear.py | 1 - hls4ml/model/optimizer/passes/resize_const.py | 27 ------------ .../passes/resize_remove_constants.py | 42 +++++++++++++++++++ test/pytest/test_qonnx.py | 22 ---------- 6 files changed, 63 insertions(+), 60 deletions(-) delete mode 100644 hls4ml/model/optimizer/passes/resize_const.py create mode 100644 hls4ml/model/optimizer/passes/resize_remove_constants.py diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index 8f9faf308b..9323da4999 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1149,27 +1149,38 @@ def initialize(self): if len(self.inputs) > 1: # get the scales of Resize node from QONNX frontend - scales = self.get_input_node(self.inputs[-1]).get_attr('value') + # see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html + scales_idx = 2 if len(self.inputs) == 3 or len(self.inputs) == 4 else 1 + scales = self.get_input_node(self.inputs[scales_idx]).get_attr('value') + if len(scales) == 4: # Resize 2D + self.set_attr('out_width', int(self.get_attr('in_width') * scales[1])) + self.set_attr('out_height', int(self.get_attr('in_height') * scales[2])) + self.set_attr('n_chan', int(self.get_attr('n_chan') * scales[3])) + elif len(scales) == 3: # Resize 1D + self.set_attr('out_width', int(self.get_attr('in_width') * scales[1])) + self.set_attr('n_chan', int(self.get_attr('n_chan') * scales[2])) + else: + raise Exception('Resize 1D and Resize 2D are the ones supported in hls4ml') if self.get_attr('data_format') == 'channels_last': if len(inp.shape) == 2: # 1D -> width + chan - shape = [int(self.get_attr('out_width') * scales[1]), int(self.get_attr('n_chan') * scales[2])] + shape = [int(self.get_attr('out_width')), int(self.get_attr('n_chan'))] dims = [f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] elif len(inp.shape) == 3: # 2D -> height + width + chan shape = [ - int(self.get_attr('out_height') * scales[1]), - int(self.get_attr('out_width') * scales[2]), - int(self.get_attr('n_chan') * scales[3]), + int(self.get_attr('out_height')), + int(self.get_attr('out_width')), + int(self.get_attr('n_chan')), ] dims = [f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}', f'N_CHAN_{self.index}'] else: if len(inp.shape) == 2: # 1D -> width + chan - shape = [int(self.get_attr('n_chan') * scales[1]), int(self.get_attr('out_width') * scales[2])] + shape = [int(self.get_attr('n_chan')), int(self.get_attr('out_width'))] dims = [f'N_CHAN_{self.index}', f'OUT_WIDTH_{self.index}'] elif len(inp.shape) == 3: # 2D -> height + width + chan shape = [ - int(self.get_attr('n_chan') * scales[1]), - int(self.get_attr('out_height') * scales[2]), - int(self.get_attr('out_width') * scales[3]), + int(self.get_attr('n_chan')), + int(self.get_attr('out_height')), + int(self.get_attr('out_width')), ] dims = [f'N_CHAN_{self.index}', f'OUT_HEIGHT_{self.index}', f'OUT_WIDTH_{self.index}'] else: diff --git a/hls4ml/model/optimizer/__init__.py b/hls4ml/model/optimizer/__init__.py index abd2af1101..3302e3c691 100644 --- a/hls4ml/model/optimizer/__init__.py +++ b/hls4ml/model/optimizer/__init__.py @@ -34,7 +34,7 @@ 'parse_qonnx', [ 'reshape_constant', - 'resize_constant', + 'resize_remove_constants', 'quant_constant_parameters', 'quant_to_activation', 'fuse_quant_with_constant', diff --git a/hls4ml/model/optimizer/passes/linear.py b/hls4ml/model/optimizer/passes/linear.py index b1aee7adc7..ce0308eb66 100644 --- a/hls4ml/model/optimizer/passes/linear.py +++ b/hls4ml/model/optimizer/passes/linear.py @@ -40,7 +40,6 @@ def transform(self, model, node): # if the activation has a quantizer (usually from a QONNX Quant node), set the previous node's output precision if quantizer is not None: prev_node.set_attr("quantizer", quantizer) - prev_node.types['result_t'] = quantizer.hls_type prev_node.get_output_variable().type.precision = quantizer.hls_type model.remove_node(node) return True diff --git a/hls4ml/model/optimizer/passes/resize_const.py b/hls4ml/model/optimizer/passes/resize_const.py deleted file mode 100644 index cd82d19d93..0000000000 --- a/hls4ml/model/optimizer/passes/resize_const.py +++ /dev/null @@ -1,27 +0,0 @@ -from hls4ml.model.layers import Constant, Resize -from hls4ml.model.optimizer import OptimizerPass - - -class ResizeConstant(OptimizerPass): - """ - To compute the output shape of resize is necessary to access the scales, that - are stored as initilizer, later on converted as constant inputs. - """ - - def match(self, node): - is_match = isinstance(node, Resize) and len(node.inputs) > 1 and node.get_input_node(node.inputs[-1]) - return is_match - - def transform(self, model, node): - """ - Remove Constant from new shape input. Note, input shape node is already used on initialize - """ - scales_node = node.get_input_node(node.inputs[-1]) - node.inputs[-1] = '' - scales_values = scales_node.get_attr('value') - node.set_attr('out_width', int(node.get_attr('in_width') * scales_values[1])) - node.set_attr('out_height', int(node.get_attr('in_height') * scales_values[2])) - if not isinstance(scales_node, Constant): - raise RuntimeError("Non-constant shape inputs are not supported") - model.remove_node(scales_node, rewire=False) - return True diff --git a/hls4ml/model/optimizer/passes/resize_remove_constants.py b/hls4ml/model/optimizer/passes/resize_remove_constants.py new file mode 100644 index 0000000000..8fb67527fd --- /dev/null +++ b/hls4ml/model/optimizer/passes/resize_remove_constants.py @@ -0,0 +1,42 @@ +from hls4ml.model.layers import Constant, Resize +from hls4ml.model.optimizer import OptimizerPass + + +class ResizeRemoveConstants(OptimizerPass): + """ + This optimizer is intended to clean the Resize node from RoI and Scales parameters that if left cause issues in hls4ml. + """ + + def match(self, node): + is_match = isinstance(node, Resize) and len(node.inputs) > 1 + return is_match + + def transform(self, model, node): + """ + Remove RoI and Scale Constant from new shape input. + """ + # see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html + scales_idx = 2 if len(node.inputs) == 3 or len(node.inputs) == 4 else 1 + scales_node = node.get_input_node(node.inputs[scales_idx]) + node.inputs[scales_idx] = '' + if not isinstance(scales_node, Constant): + raise RuntimeError("Non-constant shape inputs are not supported") + model.remove_node(scales_node, rewire=False) + if len(node.inputs) >= 3 and node.inputs[1] != '': + # RoI is present only if more than 3 inputs are specified + # RoI position is always 1 when present + roi_node = node.get_input_node(node.inputs[1]) + node.inputs[1] = '' + if not isinstance(roi_node, Constant): + raise RuntimeError("Non-constant RoI inputs are not supported") + model.remove_node(roi_node, rewire=False) + if len(node.inputs) == 4: + # Remove sizes node + sizes_node = node.get_input_node(node.inputs[-1]) + node.inputs[-1] = '' + if not isinstance(sizes_node, Constant): + raise RuntimeError("Non-constant RoI inputs are not supported") + model.remove_node(sizes_node, rewire=False) + # Clean all the '' inputs + node.inputs = list(filter(None, node.inputs)) + return True diff --git a/test/pytest/test_qonnx.py b/test/pytest/test_qonnx.py index 75dd80dcdc..f48f268626 100644 --- a/test/pytest/test_qonnx.py +++ b/test/pytest/test_qonnx.py @@ -10,7 +10,6 @@ # To conveniently run QONNX inference from qonnx.core.modelwrapper import ModelWrapper -from qonnx.transformation.base import Transformation from qonnx.transformation.channels_last import ConvertToChannelsLastAndClean from qonnx.transformation.gemm_to_matmul import GemmToMatMul @@ -336,24 +335,6 @@ def test_sep_conv(sep_conv_model, backend): np.testing.assert_allclose(y_qonnx.ravel(), y_hls4ml.ravel(), atol=1e-2, rtol=1) -class EmptyFilledRoI(Transformation): - "Remove RoI tensor of Resize node added for shape inference" - - def apply(self, model): - graph_modified = False - for node in model.graph.node: - if node.op_type == 'Resize': - # Assuming 'roi' is the second input - if len(node.input) > 2 and node.input[1] != '': - init_names = [x.name for x in model.graph.initializer] - i = init_names.index(node.input[1]) - init_to_remove = model.graph.initializer[i] - model.graph.initializer.remove(init_to_remove) - node.input[1] = '' - graph_modified = True - return (model, graph_modified) - - @pytest.mark.parametrize('backend', ['Vitis']) def test_branched_model(branched_model, backend): model = branched_model @@ -366,8 +347,6 @@ def test_branched_model(branched_model, backend): config = hls4ml.utils.config.config_from_onnx_model( model, granularity='name', backend=backend, default_precision='fixed<32,16>' ) - - model = model.transform(EmptyFilledRoI()) hls_model = hls4ml.converters.convert_from_onnx_model( model, output_dir=str(test_root_path / f'hls4mlprj_qonnx_branched_model_{backend}'), @@ -395,7 +374,6 @@ def test_tiny_unet_model(tiny_unet_model, backend): model, granularity='name', backend=backend, default_precision='fixed<32,16>' ) - model = model.transform(EmptyFilledRoI()) hls_model = hls4ml.converters.convert_from_onnx_model( model, output_dir=str(test_root_path / f'hls4mlprj_qonnx_tiny_unet_model_{backend}'), From b07e998d27de11c1334cacd0ea72b8b1a4b02efd Mon Sep 17 00:00:00 2001 From: Jovan Mitrevski Date: Fri, 15 Nov 2024 18:05:52 -0600 Subject: [PATCH 10/14] revert some unneeded changes --- hls4ml/backends/fpga/passes/clone.py | 1 - hls4ml/converters/onnx_to_hls.py | 7 ------- 2 files changed, 8 deletions(-) diff --git a/hls4ml/backends/fpga/passes/clone.py b/hls4ml/backends/fpga/passes/clone.py index 834e78e45d..a36d96dfa8 100644 --- a/hls4ml/backends/fpga/passes/clone.py +++ b/hls4ml/backends/fpga/passes/clone.py @@ -76,7 +76,6 @@ def transform(self, model, node): transformed = False for output in node.outputs: - n_outputs = len(output_map[output]) + in_output if n_outputs == 1: continue diff --git a/hls4ml/converters/onnx_to_hls.py b/hls4ml/converters/onnx_to_hls.py index 6410434625..75850fa93e 100644 --- a/hls4ml/converters/onnx_to_hls.py +++ b/hls4ml/converters/onnx_to_hls.py @@ -63,13 +63,6 @@ def get_input_shape(graph, node): """ rv = [] for inp in node.input: - # this couple of lines doesn't look very nice but I don't think it would be considered as wrong. - # It is necessary for `Resize` node, since RoI input is empty but necessary to specify also scales - # array. It might be better handled in QONNX, refers to this issue for more details: - # https://github.com/fastmachinelearning/qonnx/issues/150 - if inp == '': - rv.append([]) - continue try: value_info_idx = next((i for i, x in enumerate(graph.value_info) if x.name == inp)) dim = list(d.dim_value for d in graph.value_info[value_info_idx].type.tensor_type.shape.dim) From 354b535c97224444c5ce0502edf14e2b2ddc2e98 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Mon, 18 Nov 2024 11:14:28 +0100 Subject: [PATCH 11/14] Added some minor checks related to sizes parameter --- hls4ml/model/layers.py | 2 ++ .../model/optimizer/passes/resize_remove_constants.py | 11 ++--------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index 9323da4999..55b4b95148 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1148,6 +1148,8 @@ def initialize(self): inp = self.get_input_variable() if len(self.inputs) > 1: + if len(self.inputs) == 4: + raise Exception('Sizes parameter is not supported. Use scales instead') # get the scales of Resize node from QONNX frontend # see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html scales_idx = 2 if len(self.inputs) == 3 or len(self.inputs) == 4 else 1 diff --git a/hls4ml/model/optimizer/passes/resize_remove_constants.py b/hls4ml/model/optimizer/passes/resize_remove_constants.py index 8fb67527fd..471c1f02bf 100644 --- a/hls4ml/model/optimizer/passes/resize_remove_constants.py +++ b/hls4ml/model/optimizer/passes/resize_remove_constants.py @@ -22,21 +22,14 @@ def transform(self, model, node): if not isinstance(scales_node, Constant): raise RuntimeError("Non-constant shape inputs are not supported") model.remove_node(scales_node, rewire=False) - if len(node.inputs) >= 3 and node.inputs[1] != '': - # RoI is present only if more than 3 inputs are specified + if len(node.inputs) == 3 and node.inputs[1] != '': + # RoI is present only if 3 inputs are specified # RoI position is always 1 when present roi_node = node.get_input_node(node.inputs[1]) node.inputs[1] = '' if not isinstance(roi_node, Constant): raise RuntimeError("Non-constant RoI inputs are not supported") model.remove_node(roi_node, rewire=False) - if len(node.inputs) == 4: - # Remove sizes node - sizes_node = node.get_input_node(node.inputs[-1]) - node.inputs[-1] = '' - if not isinstance(sizes_node, Constant): - raise RuntimeError("Non-constant RoI inputs are not supported") - model.remove_node(sizes_node, rewire=False) # Clean all the '' inputs node.inputs = list(filter(None, node.inputs)) return True From 994335084c7b2d329fc8608c3dd97816f38466b7 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Mon, 18 Nov 2024 14:54:47 +0100 Subject: [PATCH 12/14] Minor fix --- hls4ml/model/layers.py | 3 +++ hls4ml/model/optimizer/passes/resize_remove_constants.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index 55b4b95148..d585a30fc9 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1148,6 +1148,9 @@ def initialize(self): inp = self.get_input_variable() if len(self.inputs) > 1: + # In order to be correctly ingested by hls4ml the QONNX resize node should have 3 inputs set with RoI left empty + if len(self.inputs) == 2: + raise Exception('RoI parameter is not supported. Use scales instead') if len(self.inputs) == 4: raise Exception('Sizes parameter is not supported. Use scales instead') # get the scales of Resize node from QONNX frontend diff --git a/hls4ml/model/optimizer/passes/resize_remove_constants.py b/hls4ml/model/optimizer/passes/resize_remove_constants.py index 471c1f02bf..0c781f1960 100644 --- a/hls4ml/model/optimizer/passes/resize_remove_constants.py +++ b/hls4ml/model/optimizer/passes/resize_remove_constants.py @@ -22,8 +22,8 @@ def transform(self, model, node): if not isinstance(scales_node, Constant): raise RuntimeError("Non-constant shape inputs are not supported") model.remove_node(scales_node, rewire=False) - if len(node.inputs) == 3 and node.inputs[1] != '': - # RoI is present only if 3 inputs are specified + if len(node.inputs) >= 2 and node.inputs[1] != '': + # RoI is present only if at least 2 inputs are specified # RoI position is always 1 when present roi_node = node.get_input_node(node.inputs[1]) node.inputs[1] = '' From 5ff517b265d78eef4046ad85b3c07e5863a08905 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Tue, 19 Nov 2024 10:51:11 +0100 Subject: [PATCH 13/14] Minor modification of the error msg --- hls4ml/model/layers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index d585a30fc9..ecc9049b73 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1150,9 +1150,14 @@ def initialize(self): if len(self.inputs) > 1: # In order to be correctly ingested by hls4ml the QONNX resize node should have 3 inputs set with RoI left empty if len(self.inputs) == 2: - raise Exception('RoI parameter is not supported. Use scales instead') + raise Exception( + 'The number of inputs to Resize node is equal to 2. ' + 'In this case, either one is trying to use a version 10 node ' + 'or one is using the RoI parameter to perform the resize operation, ' + 'both not supported in hls4ml' + ) if len(self.inputs) == 4: - raise Exception('Sizes parameter is not supported. Use scales instead') + raise Exception('Sizes parameter is not supported by hls4ml. Use scales instead') # get the scales of Resize node from QONNX frontend # see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html scales_idx = 2 if len(self.inputs) == 3 or len(self.inputs) == 4 else 1 From 6a1012922531b82548fbb97d1490a07713b53a32 Mon Sep 17 00:00:00 2001 From: nicologhielmetti Date: Wed, 20 Nov 2024 22:12:06 +0100 Subject: [PATCH 14/14] Minor fixes --- hls4ml/model/layers.py | 2 +- .../passes/resize_remove_constants.py | 21 +++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/hls4ml/model/layers.py b/hls4ml/model/layers.py index ecc9049b73..891f187eaf 100644 --- a/hls4ml/model/layers.py +++ b/hls4ml/model/layers.py @@ -1153,7 +1153,7 @@ def initialize(self): raise Exception( 'The number of inputs to Resize node is equal to 2. ' 'In this case, either one is trying to use a version 10 node ' - 'or one is using the RoI parameter to perform the resize operation, ' + 'or one is using the RoI parameter only to perform the resize operation, ' 'both not supported in hls4ml' ) if len(self.inputs) == 4: diff --git a/hls4ml/model/optimizer/passes/resize_remove_constants.py b/hls4ml/model/optimizer/passes/resize_remove_constants.py index 0c781f1960..69039c60a2 100644 --- a/hls4ml/model/optimizer/passes/resize_remove_constants.py +++ b/hls4ml/model/optimizer/passes/resize_remove_constants.py @@ -1,3 +1,5 @@ +from warnings import warn + from hls4ml.model.layers import Constant, Resize from hls4ml.model.optimizer import OptimizerPass @@ -16,20 +18,21 @@ def transform(self, model, node): Remove RoI and Scale Constant from new shape input. """ # see doc here: https://onnx.ai/onnx/operators/onnx__Resize.html - scales_idx = 2 if len(node.inputs) == 3 or len(node.inputs) == 4 else 1 + roi_index = 1 + scales_idx = 2 scales_node = node.get_input_node(node.inputs[scales_idx]) node.inputs[scales_idx] = '' if not isinstance(scales_node, Constant): raise RuntimeError("Non-constant shape inputs are not supported") model.remove_node(scales_node, rewire=False) - if len(node.inputs) >= 2 and node.inputs[1] != '': - # RoI is present only if at least 2 inputs are specified - # RoI position is always 1 when present - roi_node = node.get_input_node(node.inputs[1]) - node.inputs[1] = '' - if not isinstance(roi_node, Constant): - raise RuntimeError("Non-constant RoI inputs are not supported") - model.remove_node(roi_node, rewire=False) + # RoI position is always 1 when present + roi_node = node.get_input_node(node.inputs[roi_index]) + if roi_node.get_attr('value'): + warn('RoI value vector is not empty. Consider that RoI is not supported in hls4ml', stacklevel=2) + node.inputs[roi_index] = '' + if not isinstance(roi_node, Constant): + raise RuntimeError("Non-constant RoI inputs are not supported") + model.remove_node(roi_node, rewire=False) # Clean all the '' inputs node.inputs = list(filter(None, node.inputs)) return True