clean up mult-dimensional dense

Author: jmitrevs

hls4ml/backends/catapult/passes/pointwise.py

@@ -1,7 +1,5 @@
 from copy import copy
 
-import numpy as np
-
 from hls4ml.backends.catapult.passes.convolution_templates import (
     Conv1DConfigTemplate,
     Conv1DFunctionTemplate,
@@ -78,9 +76,6 @@ def match(self, node):
     def transform(self, model, node):
         dim = node.__class__.__name__[-2:]  # '1D' or '2D'
         pw_node = model.make_node('PointwiseConv' + dim, node.name, copy(node.attributes), node.inputs.copy())
-        if len(node.weights['weight'].data.shape) == 2:  # This can happen if we assign weights of Dense layer to 1x1 Conv2D
-            expand_axis = tuple(range(int(dim[0])))
-            pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=expand_axis)
         pw_node.weights['bias'].data = node.weights['bias'].data
         # Set strategy to ensure lowercase string is passed to the template
         if model.config.is_resource_strategy(pw_node):

hls4ml/backends/quartus/passes/pointwise.py

@@ -1,7 +1,5 @@
 from copy import copy
 
-import numpy as np
-
 from hls4ml.backends.fpga.fpga_layers import PointwiseConv1D, PointwiseConv2D
 from hls4ml.backends.quartus.passes.convolution_templates import (
     Conv1DConfigTemplate,
@@ -86,9 +84,6 @@ def transform(self, model, node):
         pw_node = model.make_node(
             'PointwiseConv' + dim, node.name, copy(node.attributes), node.inputs.copy(), outputs=node.outputs.copy()
         )
-        if len(node.weights['weight'].data.shape) == 2:  # This can happen if we assign weights of Dense layer to 1x1 Conv2D
-            expand_axis = tuple(range(int(dim[0])))
-            pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=expand_axis)
         pw_node.weights['bias'].data = node.weights['bias'].data
         model.replace_node(node, pw_node)
 

hls4ml/backends/vivado/passes/pointwise.py

@@ -1,7 +1,5 @@
 from copy import copy
 
-import numpy as np
-
 from hls4ml.backends.fpga.fpga_layers import PointwiseConv1D, PointwiseConv2D
 from hls4ml.backends.vivado.passes.convolution_templates import (
     Conv1DConfigTemplate,
@@ -78,9 +76,6 @@ def match(self, node):
     def transform(self, model, node):
         dim = node.__class__.__name__[-2:]  # '1D' or '2D'
         pw_node = model.make_node('PointwiseConv' + dim, node.name, copy(node.attributes), node.inputs.copy())
-        if len(node.weights['weight'].data.shape) == 2:  # This can happen if we assign weights of Dense layer to 1x1 Conv2D
-            expand_axis = tuple(range(int(dim[0])))
-            pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=expand_axis)
         pw_node.weights['bias'].data = node.weights['bias'].data
         # Set strategy to ensure lowercase string is passed to the template
         if model.config.is_resource_strategy(pw_node):

hls4ml/model/optimizer/__init__.py

@@ -44,6 +44,7 @@
         'qkeras_factorize_alpha',
         'extract_ternary_threshold',
         'fuse_consecutive_batch_normalization',
+        'replace_multidimensional_dense_with_conv',
     ],
 )  # TODO Maybe not all QKeras optmizers belong here?
 
@@ -53,7 +54,6 @@
         'eliminate_linear_activation',
         'fuse_consecutive_batch_normalization',
         'fuse_batch_normalization',
-        'replace_multidimensional_dense_with_conv',
         'infer_precision_types',
         'set_precision_concat',
     ],

hls4ml/model/optimizer/passes/multi_dense.py

@@ -5,14 +5,14 @@
 
 
 class ReplaceMultidimensionalDenseWithConv(OptimizerPass):
+    """
+    This matches all multidimensional Dense layers and changes them to a convolution.
+    Note:  the convolution may subsequently be changed to a pointwise convolution for
+    bakends that implement special pointwise convolutions.
+    """
+
     def match(self, node):
-        return (
-            isinstance(node, Dense)
-            and len(node.get_input_variable().shape) - sum(d == 1 for d in node.get_input_variable().shape) > 1
-        )
-        # The above sum checks for the number of dimensions in the Dense with size 1
-        # The subtraction allows the check to only count the number of dimensions with non-1 size
-        # For example, this prevents matching for a Dense layer with shape (1,N)
+        return isinstance(node, Dense) and len(node.get_input_variable().shape) > 1
 
     def transform(self, model, node):
         dim = len(node.get_input_variable().shape) - 1
@@ -23,7 +23,7 @@ def transform(self, model, node):
             'padding': 'valid',
             'n_chan': input_shape[-1],
             'n_filt': node.get_attr('n_out'),
-            'weight_data': node.get_attr('weight_data'),
+            'weight_data': np.expand_dims(node.get_attr('weight_data'), axis=tuple(range(dim))),
             'bias_data': node.get_attr('bias_data'),
         }
 
@@ -58,11 +58,8 @@ def transform(self, model, node):
         else:
             raise Exception('Cannot replace Dense over {dim}D tensor with Conv{dim}D.'.format(dim=dim))
 
-        class_name = 'PointwiseConv' + str(dim) + 'D'
+        class_name = 'Conv' + str(dim) + 'D'
         pw_node = model.make_node(class_name, node.name, pointwise_attrs, node.inputs.copy())
-        if len(node.weights['weight'].data.shape) == 2:  # This can happen if we assign weights of Dense layer to 1x1 Conv2D
-            pw_node.weights['weight'].data = np.expand_dims(node.weights['weight'].data, axis=tuple(range(dim)))
-        pw_node.weights['bias'].data = node.weights['bias'].data
         model.replace_node(node, pw_node)
 
         return True

test/pytest/test_multi_dense.py

@@ -11,46 +11,32 @@
 
 
 @pytest.mark.parametrize(
-    'backend, io_type',
+    'backend, strategy',
     [
-        ('Quartus', 'io_parallel'),
-        ('Vivado', 'io_parallel'),
-        ('Vitis', 'io_parallel'),
-        ('Vivado', 'io_stream'),
-        ('Vivado', 'io_stream'),
-        ('Vitis', 'io_stream'),
+        ('Vitis', 'Latency'),
+        ('Vitis', 'Resource'),
+        ('Quartus', 'Resource'),
+        ('Catapult', 'Latency'),
+        ('Catapult', 'Resource'),
     ],
 )
-def test_multi_dense(backend, io_type):
+@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream'])
+@pytest.mark.parametrize('shape', [(4, 3), (4, 1), (2, 3, 2), (1, 3, 1)])
+def test_multi_dense(backend, strategy, io_type, shape):
     model = tf.keras.models.Sequential()
-    model.add(
-        Dense(
-            4,
-            input_shape=(
-                8,
-                8,
-            ),
-            name='Dense',
-            use_bias=True,
-            kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10),
-            bias_initializer='zeros',
-            kernel_regularizer=None,
-            bias_regularizer=None,
-            activity_regularizer=None,
-            kernel_constraint=None,
-            bias_constraint=None,
-            activation='relu',
-        )
-    )
+    model.add(Dense(7, input_shape=shape, activation='relu'))
+    model.add(Dense(2, activation='relu'))
     model.compile(optimizer='adam', loss='mse')
 
-    X_input = np.random.rand(100, 8, 8)
+    X_input = np.random.rand(100, *shape)
+    X_input = np.round(X_input * 2**10) * 2**-10  # make it an exact ap_fixed<16,6>
 
     keras_prediction = model.predict(X_input)
 
-    default_precision = 'ap_fixed<32, 16>' if backend in ['Vivado', 'Vitis'] else 'ac_fixed<32, 16, true>'
-    config = hls4ml.utils.config_from_keras_model(model, default_precision=default_precision)
-    output_dir = str(test_root_path / f'hls4mlprj_multi_dense_{backend}_{io_type}')
+    config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend)
+    config['Model']['Strategy'] = strategy
+    shapestr = '_'.join(str(x) for x in shape)
+    output_dir = str(test_root_path / f'hls4mlprj_multi_dense_{backend}_{strategy}_{io_type}_{shapestr}')
 
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
@@ -61,5 +47,3 @@ def test_multi_dense(backend, io_type):
     hls_prediction = hls_model.predict(X_input).reshape(keras_prediction.shape)
 
     np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01)
-
-    assert list(hls_model.get_layers())[1].class_name == 'PointwiseConv1D'