Merge branch 'main' of https://github.com/fastmachinelearning/hls4ml into fifo_depth_merge

Author: thesps

README.md

@@ -8,7 +8,7 @@
 
 A package for machine learning inference in FPGAs. We create firmware implementations of machine learning algorithms using high level synthesis language (HLS). We translate traditional open-source machine learning package models into HLS that can be configured for your use-case!
 
-**Contact:** hls4ml.help@gmail.com
+If you have any questions, comments, or ideas regarding hls4ml or just want to show us how you use hls4ml, don't hesitate to reach us through the [discussions](https://github.com/fastmachinelearning/hls4ml/discussions) tab.
 
 # Documentation & Tutorial
 

docs/index.rst

@@ -35,9 +35,7 @@ Welcome to hls4ml's documentation!
 
 ``hls4ml`` is a Python package for machine learning inference in FPGAs. We create firmware implementations of machine learning algorithms using high level synthesis language (HLS). We translate traditional open-source machine learning package models into HLS that can be configured for your use-case!
 
-The project is currently in development, so please let us know if you are interested, your experiences with the package, and if you would like new features to be added.
-
-Contact: hls4ml.help@gmail.com
+The project is currently in development, so please let us know if you are interested, your experiences with the package, and if you would like new features to be added. You can reach us through our GitHub page.
 
 
 Project Status

hls4ml/backends/backend.py

@@ -25,14 +25,12 @@ def _init_class_optimizers(self):
         return class_optimizers
 
     def _init_file_optimizers(self):
-        opt_path = os.path.dirname(inspect.getfile(self.__class__)) + '/passes'
-        module_path = self.__module__[:self.__module__.rfind('.')] + '.passes'
-        file_optimizers = extract_optimizers_from_path(opt_path, module_path, self)
-        for base in self.__class__.__bases__:
-            opt_path = os.path.dirname(inspect.getfile(base)) + '/passes'
-            module_path = base.__module__[:base.__module__.rfind('.')] + '.passes'
-            base_optimizers = extract_optimizers_from_path(opt_path, module_path, self)
-            file_optimizers.update(base_optimizers)
+        file_optimizers = {}
+        for cls in [*self.__class__.__bases__, self.__class__]:
+            opt_path = os.path.dirname(inspect.getfile(cls)) + '/passes'
+            module_path = cls.__module__[:cls.__module__.rfind('.')] + '.passes'
+            cls_optimizers = extract_optimizers_from_path(opt_path, module_path, self)
+            file_optimizers.update(cls_optimizers)
         return file_optimizers
 
     def _get_layer_initializers(self):

hls4ml/backends/fpga/fpga_types.py

@@ -184,11 +184,11 @@ def definition_cpp(self, name_suffix='', as_reference=False):
 
 class VivadoArrayVariableDefinition(VariableDefinition):
     def definition_cpp(self, name_suffix='', as_reference=False):
-        return '{type} {name}{suffix}[{shape}]'.format(type=self.type.name, name=self.cppname, suffix=name_suffix, shape=self.size_cpp())
+        return '{type} {name}{suffix}[{shape}]'.format(type=self.type.name, name=self.name, suffix=name_suffix, shape=self.size_cpp())
 
 class QuartusArrayVariableDefinition(VariableDefinition):
     def definition_cpp(self, name_suffix='', as_reference=False):
-        return '{type} {name}{suffix}[{shape}] {pragma}'.format(type=self.type.name, name=self.cppname, suffix=name_suffix, shape=self.size_cpp(), pragma=self.pragma)
+        return '{type} {name}{suffix}[{shape}] {pragma}'.format(type=self.type.name, name=self.name, suffix=name_suffix, shape=self.size_cpp(), pragma=self.pragma)
 
 class ArrayVariableConverter(object):
     def __init__(self, type_converter, prefix, definition_cls):
@@ -254,9 +254,9 @@ def __init__(self, type_converter):
 class VivadoStreamVariableDefinition(VariableDefinition):
     def definition_cpp(self, name_suffix='', as_reference=False):
         if as_reference: # Function parameter
-            return 'hls::stream<{type}> &{name}{suffix}'.format(type=self.type.name, name=self.cppname, suffix=name_suffix)
+            return 'hls::stream<{type}> &{name}{suffix}'.format(type=self.type.name, name=self.name, suffix=name_suffix)
         else: # Declaration
-            return 'hls::stream<{type}> {name}{suffix}("{name}")'.format(type=self.type.name, name=self.cppname, suffix=name_suffix)
+            return 'hls::stream<{type}> {name}{suffix}("{name}")'.format(type=self.type.name, name=self.name, suffix=name_suffix)
 
 class StreamVariableConverter(object):
     def __init__(self, type_converter, prefix, definition_cls):
@@ -315,7 +315,7 @@ def __init__(self, type_converter):
 
 class StaticWeightVariableDefinition(VariableDefinition):
     def definition_cpp(self, name_suffix='', as_reference=False):
-        return '{type} {name}[{size}]'.format(type=self.type.name, name=self.cppname, size=self.data_length)
+        return '{type} {name}[{size}]'.format(type=self.type.name, name=self.name, size=self.data_length)
 
 class StaticWeightVariableConverter(object):
     def __init__(self, type_converter):

hls4ml/backends/vivado/passes/recurrent_templates.py

@@ -17,8 +17,8 @@
     typedef {bias_t.name} bias_t;
     typedef {weight_t.name} weight_t;
     typedef ap_{index_t} index_t;
-    template<class x_T, class y_T, class res_T>
-    using product = nnet::product::{product_type}<x_T, y_T, res_T>;
+    template<class x_T, class y_T>
+    using product = nnet::product::{product_type}<x_T, y_T>;
 }};\n"""
 
 #activation templates

hls4ml/backends/vivado/passes/transform_types.py

@@ -20,8 +20,6 @@ def transform(self, model, node):
                 new_var = self.inplace_var_converter.convert(var, io_type)
             if io_type == 'io_stream':
                 new_var = self.stream_var_converter.convert(var)
-            elif io_type == 'io_serial':
-                new_var = self.array_var_converter.convert(var, pragma='stream')
             elif io_type == 'io_parallel':
                 if node.name in node.model.inputs:
                     new_var = self.array_var_converter.convert(var, pragma='reshape')

hls4ml/backends/vivado/vivado_backend.py

@@ -247,9 +247,6 @@ def init_lstm(self, layer):
         reuse_factor = layer.model.config.get_reuse_factor(layer)
         layer.set_attr('recurrent_reuse_factor', reuse_factor)
 
-        recurrent_bias = np.zeros(layer.weights['recurrent_weight'].shape[1])
-        layer.add_weights_variable(name='recurrent_bias', var_name='br{index}', data=recurrent_bias)
-
         index_t = IntegerPrecisionType(width=1, signed=False)
 
         if 'table_t' not in layer.attributes:
@@ -273,9 +270,6 @@ def init_gru(self, layer):
         reuse_factor = layer.model.config.get_reuse_factor(layer)
         layer.set_attr('recurrent_reuse_factor', reuse_factor)
 
-        recurrent_bias = np.zeros(layer.weights['recurrent_weight'].shape[1])
-        layer.add_weights_variable(name='recurrent_bias', var_name='br{index}', data=recurrent_bias)
-
         index_t = IntegerPrecisionType(width=1, signed=False)
 
         if 'table_t' not in layer.attributes:

hls4ml/converters/__init__.py

@@ -188,7 +188,7 @@ def convert_from_keras_model(model, output_dir='my-hls-test', project_name='mypr
         clock_period (int, optional): Clock period of the design.
             Defaults to 5.
         io_type (str, optional): Type of implementation used. One of
-            'io_parallel' or 'io_serial'. Defaults to 'io_parallel'.
+            'io_parallel' or 'io_stream'. Defaults to 'io_parallel'.
         hls_config (dict, optional): The HLS config.
         kwargs** (dict, optional): Additional parameters that will be used to create the config of the specified backend
     Raises:
@@ -246,7 +246,7 @@ def convert_from_pytorch_model(model, input_shape, output_dir='my-hls-test', pro
     clock_period (int, optional): Clock period of the design.
         Defaults to 5.
     io_type (str, optional): Type of implementation used. One of
-        'io_parallel' or 'io_serial'. Defaults to 'io_parallel'.
+        'io_parallel' or 'io_stream'. Defaults to 'io_parallel'.
     hls_config (dict, optional): The HLS config.
     kwargs** (dict, optional): Additional parameters that will be used to create the config of the specified backend
 
@@ -319,7 +319,7 @@ def convert_from_onnx_model(model, output_dir='my-hls-test', project_name='mypro
     clock_period (int, optional): Clock period of the design.
         Defaults to 5.
     io_type (str, optional): Type of implementation used. One of
-        'io_parallel' or 'io_serial'. Defaults to 'io_parallel'.
+        'io_parallel' or 'io_stream'. Defaults to 'io_parallel'.
     hls_config (dict, optional): The HLS config.
     kwargs** (dict, optional): Additional parameters that will be used to create the config of the specified backend
 

hls4ml/converters/keras/reshaping.py

@@ -26,6 +26,8 @@ def parse_zeropadding1d_layer(keras_layer, input_names, input_shapes, data_reade
         ]
         layer['out_width'] = output_shape[2]
         layer['n_chan'] = output_shape[1]
+
+        layer['in_width'] = input_shapes[0][2]
     else:
         output_shape = [
             input_shapes[0][0], # Batch
@@ -35,6 +37,8 @@ def parse_zeropadding1d_layer(keras_layer, input_names, input_shapes, data_reade
         layer['out_width'] = output_shape[1]
         layer['n_chan'] = output_shape[2]
 
+        layer['in_width'] = input_shapes[0][1]
+
     return layer, output_shape
 
 @keras_handler('ZeroPadding2D')
@@ -74,6 +78,9 @@ def parse_zeropadding2d_layer(keras_layer, input_names, input_shapes, data_reade
         layer['out_height'] = output_shape[2]
         layer['out_width'] = output_shape[3]
         layer['n_chan'] = output_shape[1]
+
+        layer['in_height'] = input_shapes[0][2]
+        layer['in_width'] = input_shapes[0][3]
     else:
         output_shape = [
             input_shapes[0][0], # Batch
@@ -85,4 +92,7 @@ def parse_zeropadding2d_layer(keras_layer, input_names, input_shapes, data_reade
         layer['out_width'] = output_shape[2]
         layer['n_chan'] = output_shape[3]
 
+        layer['in_height'] = input_shapes[0][1]
+        layer['in_width'] = input_shapes[0][2]
+
     return layer, output_shape

hls4ml/model/layers.py

@@ -885,12 +885,17 @@ def initialize(self):
             self.add_output_variable(state_shape, state_dims, out_name=self.outputs[1], var_name='layer{index}_h', type_name='layer{index}_h_t')
             self.add_output_variable(state_shape, state_dims, out_name=self.outputs[2], var_name='layer{index}_c', type_name='layer{index}_c_t')
 
+        #weights
         self.add_weights()
-        self.add_bias()
 
+        #recurrent weights
         recurrent_weight = self.model.get_weights_data(self.name, 'recurrent_kernel')
         self.add_weights_variable(name='recurrent_weight', var_name='wr{index}', data=recurrent_weight)
 
+        #biases
+        biases = self.model.get_weights_data(self.name , 'bias')
+        self.add_weights_variable(name='bias', var_name='b{index}', data=biases)
+
 class LSTM(Layer):
     _expected_attributes = [
         Attribute('n_out'),
@@ -904,10 +909,12 @@ class LSTM(Layer):
         WeightAttribute('weight'),
         WeightAttribute('bias'),
         WeightAttribute('recurrent_weight'),
+        WeightAttribute('recurrent_bias'),
 
         TypeAttribute('weight'),
         TypeAttribute('bias'),
         TypeAttribute('recurrent_weight'),
+        TypeAttribute('recurrent_bias'),
     ]
 
     def initialize(self):
@@ -926,12 +933,20 @@ def initialize(self):
             self.add_output_variable(state_shape, state_dims, out_name=self.outputs[1], var_name='layer{index}_h', type_name='layer{index}_h_t')
             self.add_output_variable(state_shape, state_dims, out_name=self.outputs[2], var_name='layer{index}_c', type_name='layer{index}_c_t')
 
+        #weights
         self.add_weights()
-        self.add_bias()
 
+        #recurrent weights
         recurrent_weight = self.model.get_weights_data(self.name, 'recurrent_kernel')
         self.add_weights_variable(name='recurrent_weight', var_name='wr{index}', data=recurrent_weight)
 
+        #biases
+        biases = self.model.get_weights_data(self.name , 'bias')
+        self.add_weights_variable(name='bias', var_name='b{index}', data=biases)
+
+        recurrent_bias = np.zeros(recurrent_weight.shape[1])
+        self.add_weights_variable(name='recurrent_bias', var_name='br{index}', data=recurrent_bias)
+
 class GRU(Layer):
     _expected_attributes = [
         Attribute('n_out'),
@@ -946,10 +961,12 @@ class GRU(Layer):
         WeightAttribute('weight'),
         WeightAttribute('bias'),
         WeightAttribute('recurrent_weight'),
+        WeightAttribute('recurrent_bias'),
 
         TypeAttribute('weight'),
         TypeAttribute('bias'),
         TypeAttribute('recurrent_weight'),
+        TypeAttribute('recurrent_bias'),
     ]
 
     def initialize(self):
@@ -968,12 +985,19 @@ def initialize(self):
             self.add_output_variable(state_shape, state_dims, out_name=self.outputs[1], var_name='layer{index}_h', type_name='layer{index}_h_t')
             self.add_output_variable(state_shape, state_dims, out_name=self.outputs[2], var_name='layer{index}_c', type_name='layer{index}_c_t')
 
+        #weights
         self.add_weights()
-        self.add_bias()
 
+        #recurrent weights
         recurrent_weight = self.model.get_weights_data(self.name, 'recurrent_kernel')
         self.add_weights_variable(name='recurrent_weight', var_name='wr{index}', data=recurrent_weight)
 
+        #biases array is actually a 2-dim array of arrays (bias + recurrent bias)
+        #both arrays have shape: n_units * 3 (z, r, h_cand)
+        biases = self.model.get_weights_data(self.name , 'bias')
+        self.add_weights_variable(name='bias', var_name='b{index}', data=biases[0])
+        self.add_weights_variable(name='recurrent_bias', var_name='br{index}', data=biases[1])
+
 class GarNet(Layer):
     ref_impl = False