fix merge conflicts in RNN parsing

Author: JanFSchulte

hls4ml/backends/vivado/passes/convolution_templates.py

@@ -1,6 +1,14 @@
 from hls4ml.backends.backend import get_backend
 from hls4ml.backends.template import FunctionCallTemplate, LayerConfigTemplate
-from hls4ml.model.layers import Conv1D, Conv2D, Conv2DBatchnorm, DepthwiseConv2D, SeparableConv1D, SeparableConv2D
+from hls4ml.model.layers import (
+    Conv1D,
+    Conv2D,
+    Conv2DBatchnorm,
+    DepthwiseConv1D,
+    DepthwiseConv2D,
+    SeparableConv1D,
+    SeparableConv2D,
+)
 
 # Shared multiplication template
 
@@ -52,13 +60,16 @@
 const ap_uint<config{index}::filt_width> config{index}::pixels[] = {{{instructions}}};\n"""
 
 conv1d_function_template = 'nnet::conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
+depthconv1d_function_template = (
+    'nnet::depthwise_conv_1d_{data_format}<{input_t}, {output_t}, {config}>({input}, {output}, {w}, {b});'
+)
 
 conv1d_include_list = ['nnet_utils/nnet_conv1d.h', 'nnet_utils/nnet_conv1d_stream.h']
 
 
 class Conv1DConfigTemplate(LayerConfigTemplate):
     def __init__(self):
-        super().__init__(Conv1D)
+        super().__init__((Conv1D, DepthwiseConv1D))
         self.template = conv1d_config_template
         self.mult_template = conv_mult_config_template
 
@@ -106,6 +117,12 @@ def format(self, node):
         return self.template.format(**params)
 
 
+class DepthwiseConv1DFunctionTemplate(Conv1DFunctionTemplate):
+    def __init__(self):
+        super(Conv1DFunctionTemplate, self).__init__(DepthwiseConv1D, include_header=sepconv1d_include_list)
+        self.template = depthconv1d_function_template
+
+
 # Conv2D Templates
 
 conv2d_config_template = """struct config{index} : nnet::conv2d_config {{

hls4ml/converters/keras/convolution.py

@@ -2,7 +2,7 @@
 from hls4ml.converters.utils import compute_padding_1d, compute_padding_2d, parse_data_format
 
 
-@keras_handler('Conv1D', 'SeparableConv1D')
+@keras_handler('Conv1D', 'SeparableConv1D', 'DepthwiseConv1D')
 def parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader):
     assert 'Conv1D' in keras_layer['class_name']
 
@@ -12,14 +12,19 @@ def parse_conv1d_layer(keras_layer, input_names, input_shapes, data_reader):
 
     if layer['class_name'] in ['Conv1D', 'QConv1D']:
         layer['weight_data'] = get_weights_data(data_reader, layer['name'], 'kernel')
-    else:  # SeparableConv1D
-        layer['depthwise_data'], layer['pointwise_data'], layer['bias_data'] = get_weights_data(
-            data_reader, layer['name'], ['depthwise_kernel', 'pointwise_kernel', 'bias']
+    elif layer['class_name'] in ['SeparableConv1D', 'QSeparableConv1D']:
+        layer['depthwise_data'], layer['pointwise_data'] = get_weights_data(
+            data_reader, layer['name'], ['depthwise_kernel', 'pointwise_kernel']
         )
+    else:  # DepthwiseConv1D
+        layer['depthwise_data'] = get_weights_data(data_reader, layer['name'], 'depthwise_kernel')
 
     layer['bias_data'] = get_weights_data(data_reader, layer['name'], 'bias')
 
-    layer['n_filt'] = keras_layer['config']['filters']
+    if 'filters' in keras_layer['config']:
+        layer['n_filt'] = keras_layer['config']['filters']
+    else:
+        layer['n_filt'] = layer['n_chan']
     layer['filt_width'] = keras_layer['config']['kernel_size'][0]
     layer['stride_width'] = keras_layer['config']['strides'][0]
     layer['padding'] = keras_layer['config']['padding']

hls4ml/converters/keras/qkeras.py

@@ -49,6 +49,19 @@ def parse_qconv_layer(keras_layer, input_names, input_shapes, data_reader):
     return layer, output_shape
 
 
+@keras_handler('QDepthwiseConv2D')
+def parse_qdepthwiseqconv_layer(keras_layer, input_names, input_shapes, data_reader):
+    layer, output_shape = parse_conv2d_layer(keras_layer, input_names, input_shapes, data_reader)
+
+    layer['depthwise_quantizer'] = get_quantizer_from_config(keras_layer, 'depthwise')
+    if keras_layer['config']['bias_quantizer'] is not None:
+        layer['bias_quantizer'] = get_quantizer_from_config(keras_layer, 'bias')
+    else:
+        layer['bias_quantizer'] = None
+
+    return layer, output_shape
+
+
 @keras_handler('QActivation')
 def parse_qactivation_layer(keras_layer, input_names, input_shapes, data_reader):
     assert keras_layer['class_name'] == 'QActivation'

hls4ml/converters/keras_to_hls.py

@@ -301,7 +301,7 @@ def parse_keras_model(model_arch, reader):
                 act_layer['class_name'] = 'QActivation'
                 act_layer['config'] = {
                     'name': layer['name'] + '_' + act_details['class_name'],
-                    'activation': act_details['class_name'],
+                    'activation': act_details,
                 }
                 act_layer, output_shape = layer_handlers['QActivation'](act_layer, None, [output_shape], reader)
             else:

hls4ml/converters/pytorch/convolution.py

@@ -1,4 +1,4 @@
-from hls4ml.converters.pytorch_to_hls import get_weights_data, pytorch_handler
+from hls4ml.converters.pytorch_to_hls import pytorch_handler
 from hls4ml.converters.utils import compute_padding_1d_pytorch, compute_padding_2d_pytorch, parse_data_format
 
 
@@ -9,11 +9,16 @@ def parse_conv1d_layer(operation, layer_name, input_names, input_shapes, node, c
     layer = {}
 
     layer['name'] = layer_name
+    layer['inputs'] = input_names
     layer['class_name'] = 'Conv1D'
     layer['data_format'] = 'channels_first'  # Pytorch default (can't change)
 
-    layer['weight_data'] = get_weights_data(data_reader, layer['name'], 'weight')
-    layer['bias_data'] = get_weights_data(data_reader, layer['name'], 'bias')
+    layer['weight_data'] = class_object.weight.data.numpy()
+    if class_object.bias is not None:
+        layer['bias_data'] = class_object.bias.data.numpy()
+    else:
+        layer['bias_data'] = None
+
     # Input info
     (layer['in_width'], layer['n_chan']) = parse_data_format(
         input_shapes[0], 'channels_first'
@@ -54,11 +59,16 @@ def parse_conv2d_layer(operation, layer_name, input_names, input_shapes, node, c
     layer = {}
 
     layer['name'] = layer_name
+    layer['inputs'] = input_names
     layer['class_name'] = 'Conv2D'
     layer['data_format'] = 'channels_first'  # Pytorch default (can't change)
 
-    layer['weight_data'] = get_weights_data(data_reader, layer['name'], 'weight')
-    layer['bias_data'] = get_weights_data(data_reader, layer['name'], 'bias')
+    layer['weight_data'] = class_object.weight.data.numpy()
+    if class_object.bias is not None:
+        layer['bias_data'] = class_object.bias.data.numpy()
+    else:
+        layer['bias_data'] = None
+
     # Input info
     (layer['in_height'], layer['in_width'], layer['n_chan']) = parse_data_format(
         input_shapes[0], 'channels_first'

hls4ml/converters/pytorch/core.py

@@ -1,4 +1,4 @@
-from hls4ml.converters.pytorch_to_hls import get_weights_data, pytorch_handler
+from hls4ml.converters.pytorch_to_hls import pytorch_handler
 
 
 @pytorch_handler('Linear')
@@ -9,8 +9,14 @@ def parse_linear_layer(operation, layer_name, input_names, input_shapes, node, c
 
     layer['class_name'] = 'Dense'
     layer['name'] = layer_name
+    layer['inputs'] = input_names
+
+    layer['weight_data'] = class_object.weight.data.numpy()
+    if class_object.bias is not None:
+        layer['bias_data'] = class_object.bias.data.numpy()
+    else:
+        layer['bias_data'] = None
 
-    layer['weight_data'], layer['bias_data'] = get_weights_data(data_reader, layer['name'], ['weight', 'bias'])
     if class_object is not None:
         layer['n_in'] = class_object.in_features
         layer['n_out'] = class_object.out_features
@@ -39,6 +45,7 @@ def parse_activation_layer(operation, layer_name, input_names, input_shapes, nod
     layer['class_name'] = operation
     layer['activation'] = layer['class_name']
     layer['name'] = layer_name
+    layer['inputs'] = input_names
 
     # if layer['class_name'] != 'Activation':
     #    layer['activation'] = layer['class_name']
@@ -50,7 +57,7 @@ def parse_activation_layer(operation, layer_name, input_names, input_shapes, nod
         if layer['class_name'] == 'ELU':
             layer['activ_param'] = class_object.alpha
         if layer['class_name'] == 'PReLU':
-            layer['alpha_data'] = get_weights_data(data_reader, layer['name'], 'weight')
+            layer['alpha_data'] = class_object.weight.data.numpy()
         if layer['class_name'] == 'Threshold':
             layer['activ_param'] = class_object.threshold
             layer['class_name'] = 'ThresholdedReLU'
@@ -92,25 +99,26 @@ def parse_batchnorm_layer(operation, layer_name, input_names, input_shapes, node
     layer['class_name'] = 'BatchNormalization'
     layer['data_format'] = 'channels_first'
     layer['name'] = layer_name
+    layer['inputs'] = input_names
 
     # batchnorm para
     if node.op == 'call_module':
         layer['epsilon'] = class_object.eps
         layer['use_gamma'] = layer['use_beta'] = class_object.affine
 
         if layer['use_gamma']:
-            layer['gamma_data'] = get_weights_data(data_reader, layer['name'], 'weight')
+            layer['gamma_data'] = class_object.weight.data.numpy()
         else:
             layer['gamma_data'] = 1
 
         if layer['use_beta']:
-            layer['beta_data'] = get_weights_data(data_reader, layer['name'], 'bias')
+            layer['beta_data'] = class_object.bias.data.numpy()
         else:
             layer['beta_data'] = 0
 
-        layer['mean_data'], layer['variance_data'] = get_weights_data(
-            data_reader, layer['name'], ['running_mean', 'running_var']
-        )
+        layer['mean_data'] = class_object.running_mean.data.numpy()
+        layer['variance_data'] = class_object.running_var.data.numpy()
+
     in_size = 1
     for dim in input_shapes[0][1:]:
         in_size *= dim

hls4ml/converters/pytorch/pooling.py

@@ -20,6 +20,7 @@ def parse_pooling_layer(operation, layer_name, input_names, input_shapes, node,
         layer['class_name'] = 'AveragePooling2D'
 
     layer['name'] = layer_name
+    layer['inputs'] = input_names
     layer['data_format'] = 'channels_first'  # Pytorch default (can't change)
     if node.op == 'call_module' and 'Avg' in operation:
         if class_object.count_include_pad:

hls4ml/converters/pytorch/recurrent.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from hls4ml.converters.pytorch_to_hls import get_weights_data, pytorch_handler
+from hls4ml.converters.pytorch_to_hls import pytorch_handler
 
 rnn_layers = ['RNN', 'LSTM', 'GRU']
 
@@ -55,12 +55,10 @@ def parse_rnn_layer(operation, layer_name, input_names, input_shapes, node, clas
     if class_object.dropout > 0:
         raise Exception('hls4ml does not support RNNs with dropout')
 
-    (
-        layer['weight_data'],
-        layer['recurrent_weight_data'],
-        layer['bias_data'],
-        layer['recurrent_bias_data'],
-    ) = get_weights_data(data_reader, layer['name'], ['weight_ih_l0', 'weight_hh_l0', 'bias_ih_l0', 'bias_hh_l0'])
+    layer['weight_data'] = class_object.weight_ih_l0.data.numpy()
+    layer['recurrent_weight_data'] = class_object.weight_hh_l0.data.numpy()
+    layer['bias_data'] = class_object.bias_ih_l0.data.numpy()
+    layer['recurrent_bias_data'] = class_object.bias_hh_l0.data.numpy()
 
     if class_object.bias is False:
         layer['bias_data'] = np.zeros(layer['weight_data'].shape[0])

hls4ml/converters/pytorch/reshape.py

@@ -12,6 +12,7 @@ def parse_reshape_layer(operation, layer_name, input_names, input_shapes, node,
     layer = {}
     layer['class_name'] = 'Reshape'
     layer['name'] = layer_name
+    layer['inputs'] = input_names
 
     layer['target_shape'] = [int(i) for i in node.args[1:]]
     # View can have -1 as one as the dimensions,
@@ -27,3 +28,81 @@ def parse_reshape_layer(operation, layer_name, input_names, input_shapes, node,
     output_shape = input_shapes[0][:1] + layer['target_shape']
 
     return layer, output_shape
+
+
+@pytorch_handler('squeeze')
+def parse_squeeze_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config):
+    assert operation == 'squeeze'
+
+    layer = {}
+    layer['class_name'] = 'Reshape'
+    layer['name'] = layer_name
+
+    if len(node.args) > 1 or len(node.kwargs) > 0:  # 'dim' argument is specified
+        output_shape = [i for i in input_shapes[0]]
+        squeeze_dim = node.kwargs.get('dim', None)
+        if squeeze_dim is None:
+            squeeze_dim = node.args[1]
+        if isinstance(squeeze_dim, tuple):
+            for dim in squeeze_dim:
+                del output_shape[dim]
+        else:
+            del output_shape[squeeze_dim]
+    else:
+        output_shape = [i for i in input_shapes[0] if i != 1]
+
+    layer['target_shape'] = output_shape.copy()
+    if layer['target_shape'][0] is None:
+        del layer['target_shape'][0]
+
+    return layer, output_shape
+
+
+@pytorch_handler('unsqueeze')
+def parse_unsqueeze_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config):
+    assert operation == 'unsqueeze'
+
+    layer = {}
+    layer['class_name'] = 'Reshape'
+    layer['name'] = layer_name
+    layer['inputs'] = input_names
+
+    # Unlike in 'squeeze' in 'unsqueeze', dim argument must exist
+    output_shape = [i for i in input_shapes[0]]
+    if len(node.args) > 1:  # Specified as unsqueeze(x, n)
+        squeeze_dim = node.args[1]
+    else:  # Specified as unsqueeze(x, dim=n)
+        squeeze_dim = node.kwargs['dim']
+    # insert() will add an element before the index, unsqueeze expects the location
+    index = output_shape.index(output_shape[squeeze_dim])  # + 1
+    output_shape.insert(index, 1)
+
+    layer['target_shape'] = output_shape.copy()
+    if layer['target_shape'][0] is None:
+        del layer['target_shape'][0]
+
+    return layer, output_shape
+
+
+@pytorch_handler('Flatten')
+def parse_flatten_layer(operation, layer_name, input_names, input_shapes, node, class_object, data_reader, config):
+    assert operation == 'Flatten'
+
+    layer = {}
+    layer['class_name'] = 'Reshape'
+    layer['name'] = layer_name
+    layer['inputs'] = input_names
+
+    start_dim = class_object.start_dim
+    end_dim = class_object.end_dim
+    if end_dim + 1 == 0 or end_dim + 1 > len(input_shapes[0]):
+        end_dim = len(input_shapes[0])
+    else:
+        end_dim = end_dim + 1
+
+    layer['target_shape'] = (
+        input_shapes[0][0:start_dim] + [np.prod(input_shapes[0][start_dim:end_dim])] + input_shapes[0][end_dim:]
+    )
+    output_shape = layer['target_shape']
+
+    return layer, output_shape