change 2d transpose weight input to be 3-dimensional (passed from python code)

Author: Jonathan-Shoemaker

hls4ml/backends/vivado/passes/resource_strategy.py

@@ -7,7 +7,7 @@ class ApplyResourceStrategy(OptimizerPass):
     ''' Transposes the weights to use the dense_resource matrix multiply routine '''
     def match(self, node):
 
-        node_matches = isinstance(node, (Dense, Conv1D, SeparableConv1D, Conv2D, Conv1DTranspose, Conv2DTranspose, SeparableConv2D, LSTM, GRU))
+        node_matches = isinstance(node, (Dense, Conv1D, SeparableConv1D, Conv2D, SeparableConv2D, LSTM, GRU))
 
         is_resource_strategy = node.get_attr('strategy', '').lower() == 'resource'
         already_transformed = node.get_attr('_weights_transposed', False) == True
@@ -19,30 +19,11 @@ def transform(self, model, node):
             node.weights['weight'].data = np.transpose(node.weights['weight'].data)
         elif isinstance(node, Conv1D):
             node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[2, 0, 1]) #(W,C,F) => (F,W,C)
-        elif isinstance(node, Conv1DTranspose):
-            pass
-        #     #(W,F,C) => (F,W,C)
-        #     node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[1, 0, 2]) 
-        #     # now split the kernel into stride width kernels (F, W, C) -> (S, F, W/S, C)
-        #     n_filts, kern_width, n_chan = node.weights['weight'].data.shape 
-        #     new_weights = np.zeros((node.get_attr('stride_width'), n_filts, node.get_attr('trfilt_width'), n_chan))
-        #     for i_sw in range(node.get_attr('stride_width')):
-        #         for i_fw in range(node.get_attr('trfilt_width')):
-        #             filt_ind = i_sw + (node.get_attr('trfilt_width')-i_fw-1) * node.get_attr('stride_width')
-        #             for i_nf in range(n_filts):
-        #                 for i_nc in range(n_chan):
-        #                     if filt_ind < kern_width:
-        #                         new_weights[i_sw][i_nf][i_fw][i_nc] = \
-        #                             node.weights['weight'].data[i_nf][filt_ind][i_nc]
-        #     node.weights['weight'].data = new_weights
-        #     print("Updated shape:", node.weights['weight'].data.shape)
         elif isinstance(node, SeparableConv1D):
             node.weights['depthwise'].data = np.transpose(node.weights['depthwise'].data, axes=[2, 0, 1]) #(W,C,F) => (F,W,C)
             node.weights['pointwise'].data = np.transpose(node.weights['pointwise'].data, axes=[2, 0, 1]) #(W,C,F) => (F,W,C)
         elif isinstance(node, Conv2D):
             node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[3, 0, 1, 2]) #(H,W,C,F) => (F,H,W,C)
-        elif isinstance(node, Conv2DTranspose):
-            node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[2, 0, 1, 3]) #(H,W,F,C) => (F,H,W,C)
         elif isinstance(node, SeparableConv2D):
             node.weights['depthwise'].data = np.transpose(node.weights['depthwise'].data, axes=[3, 0, 1, 2]) #(H,W,C,F) => (F,H,W,C)
             node.weights['pointwise'].data = np.transpose(node.weights['pointwise'].data, axes=[3, 0, 1, 2]) #(H,W,C,F) => (F,H,W,C)

hls4ml/converters/keras/convolution.py

@@ -138,6 +138,8 @@ def parse_conv2dtranspose_layer(keras_layer, input_names, input_shapes, data_rea
     layer['stride_height'] = keras_layer['config']['strides'][0]
     layer['stride_width'] = keras_layer['config']['strides'][1]
     layer['padding'] = keras_layer['config']['padding']
+    layer['trfilt_height'] = (layer['filt_height'] + layer['stride_height'] - 1)//layer['stride_height']
+    layer['trfilt_width'] = (layer['filt_width'] + layer['stride_width'] - 1)//layer['stride_width']
 
     (
         layer['out_height'],

hls4ml/model/layers.py

@@ -509,8 +509,32 @@ def initialize(self):
         else:
             shape = [self.attributes['n_filt'], self.attributes['out_height'], self.attributes['out_width']]
             dims = ['N_FILT_{}'.format(self.index), 'OUT_HEIGHT_{}'.format(self.index), 'OUT_WIDTH_{}'.format(self.index)]
+
+        data = self.model.get_weights_data(self.name, 'kernel')
+        # now we transform the entire kernel
+
+        #(H,W,F,C) => (F,H,W,C)
+        data = np.transpose(data, axes=[2, 0, 1, 3]) 
+        # now split the kernel into stride width kernels (F, W, C) -> (Sh, Sw, F, H/Sh, W/Sw, C)
+        n_filts, kern_height, kern_width, n_chan = data.shape
+        new_weights = np.zeros((self.attributes['stride_height'], self.attributes['stride_width'], \
+            n_filts, self.attributes['trfilt_height'], self.attributes['trfilt_width'], n_chan))
+        for i_sh in range(self.attributes['stride_height']):
+            for i_sw in range(self.attributes['stride_width']):
+                for i_fh in range(self.attributes['trfilt_height']):
+                    for i_fw in range(self.attributes['trfilt_width']):
+                        filt_h_ind = i_sh + (self.attributes['trfilt_height']-i_fh-1)*self.attributes['stride_height']
+                        filt_w_ind = i_sw + (self.attributes['trfilt_width']-i_fw-1)*self.attributes['stride_width']
+                        for i_nf in range(n_filts):
+                            for i_nc in range(n_chan):
+                                if filt_h_ind < kern_height and filt_w_ind < kern_width:
+                                    new_weights[i_sh][i_sw][i_nf][i_fh][i_fw][i_nc] = \
+                                        data[i_nf][filt_h_ind][filt_w_ind][i_nc]
+        data = new_weights
+
         self.add_output_variable(shape, dims)
-        self.add_weights(quantizer=self.get_attr('weight_quantizer'))
+        self.add_weights_variable(name='weight', var_name='w{index}', \
+            data=data, quantizer=self.get_attr('weight_quantizer'), keep_dims=2)
         self.add_bias(quantizer=self.get_attr('bias_quantizer'))
 
 

hls4ml/templates/vivado/nnet_utils/nnet_conv2dtranspose.h

@@ -44,7 +44,9 @@ template<class data_T, class res_T, typename CONFIG_T>
 void conv_2d_transpose_cl(
     data_T data[CONFIG_T::in_height * CONFIG_T::in_width * CONFIG_T::n_chan],
     res_T  res[CONFIG_T::out_height * CONFIG_T::out_width * CONFIG_T::n_filt],
-    typename CONFIG_T::weight_t weights[CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan * CONFIG_T::n_filt],
+    typename CONFIG_T::weight_t weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
+        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
+    ],
     typename CONFIG_T::bias_t   biases[CONFIG_T::n_filt]
 )
 {

hls4ml/templates/vivado/nnet_utils/nnet_conv2dtranspose_resource.h

@@ -10,7 +10,9 @@ template<class data_T, class res_T, typename CONFIG_T>
 void conv_2d_transpose_resource_cl(
     data_T data[CONFIG_T::in_height * CONFIG_T::in_width * CONFIG_T::n_chan],
     res_T  res[CONFIG_T::out_height * CONFIG_T::out_width * CONFIG_T::n_filt],
-    typename CONFIG_T::weight_t weights[CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan * CONFIG_T::n_filt],
+    typename CONFIG_T::weight_t weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
+        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
+    ],
     typename CONFIG_T::bias_t   biases[CONFIG_T::n_filt]
 )
 {
@@ -32,51 +34,7 @@ void conv_2d_transpose_resource_cl(
     typename CONFIG_T::accum_t acc[CONFIG_T::n_pixels][mult_n_out][CONFIG_T::stride_height][CONFIG_T::stride_width];
     #pragma HLS ARRAY_PARTITION variable=acc complete dim=0
 
-    typename CONFIG_T::weight_t trfilt_weights[
-        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
-    ][CONFIG_T::stride_height][CONFIG_T::stride_width];
-
-    //pull out the individual filter weights (split kernel into stride_height x stride_width kernels)
-    for (unsigned i_sh = 0; i_sh < CONFIG_T::stride_height; i_sh++) {
-        #pragma HLS UNROLL
-        for (unsigned i_sw = 0; i_sw < CONFIG_T::stride_width; i_sw++) {
-            #pragma HLS UNROLL
-            for (unsigned i_fh = 0; i_fh < CONFIG_T::trfilt_height; i_fh++) {
-                #pragma HLS UNROLL
-                for (unsigned i_fw = 0; i_fw < CONFIG_T::trfilt_width; i_fw++) {
-                    #pragma HLS UNROLL
-                    unsigned filt_h_ind = i_sh + (CONFIG_T::trfilt_height-i_fh-1)*CONFIG_T::stride_height;
-                    unsigned filt_w_ind = i_sw + (CONFIG_T::trfilt_width-i_fw-1)*CONFIG_T::stride_width;
-                    for (unsigned i_nf = 0; i_nf < CONFIG_T::n_filt; i_nf++) {
-                        #pragma HLS UNROLL
-                        for (unsigned i_nc = 0; i_nc < CONFIG_T::n_chan; i_nc++) {
-                            #pragma HLS UNROLL
-                            if (filt_h_ind < CONFIG_T::filt_height && filt_w_ind < CONFIG_T::filt_width) {
-                                trfilt_weights[
-                                    i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_height * CONFIG_T::trfilt_width + 
-                                    i_fh * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
-                                    i_fw * CONFIG_T::n_chan + i_nc
-                                ][i_sh][i_sw] = weights[
-                                    i_nf * CONFIG_T::n_chan * CONFIG_T::filt_height * CONFIG_T::filt_width + 
-                                    filt_h_ind * CONFIG_T::n_chan * CONFIG_T::filt_width + 
-                                    filt_w_ind * CONFIG_T::n_chan + i_nc
-                                ];
-                            }
-                            else {
-                                trfilt_weights[
-                                    i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_height * CONFIG_T::trfilt_width + 
-                                    i_fh * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
-                                    i_fw * CONFIG_T::n_chan + i_nc
-                                ][i_sh][i_sw] = 0;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-
-    #pragma HLS ARRAY_RESHAPE variable=trfilt_weights block factor=block_factor dim=1
+    #pragma HLS ARRAY_RESHAPE variable=weights block factor=block_factor dim=3
 
     PartitionLoop:
     for (unsigned i_part = 0; i_part < CONFIG_T::n_partitions; i_part++) {
@@ -127,7 +85,7 @@ void conv_2d_transpose_resource_cl(
 
                             acc[i_pxl][i_out][i_sh][i_sw] += static_cast<typename CONFIG_T::accum_t>(
                                 CONFIG_T::mult_config::template product<data_T, typename CONFIG_T::mult_config::weight_t>::product(
-                                    data_buf[i_pxl][i_in], trfilt_weights[i_w][i_sh][i_sw]
+                                    data_buf[i_pxl][i_in], weights[i_sh][i_sw][i_w]
                                 )
                             );
                         }

hls4ml/templates/vivado/nnet_utils/nnet_conv2dtranspose_stream.h

@@ -68,64 +68,14 @@ void shift_line_buffer_tr(const data_T& in_elem,
     kernel_shift_tr_2d<data_T, CONFIG_T>(shift_buffer, kernel_window);
 }
 
-template<typename CONFIG_T>
-void load_trfilt_weights(
-    typename CONFIG_T::weight_t trfilt_weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
-        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
-    ],
-    typename CONFIG_T::weight_t weights[
-        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_chan * CONFIG_T::n_filt
-    ]
-)
-{
-    #pragma HLS INLINE
-    //pull out the individual filter weights (split kernel into stride_height x stride_width kernels)
-    TrfiltWeightsLoop: for (unsigned i_sh = 0; i_sh < CONFIG_T::stride_height; i_sh++) {
-        #pragma HLS UNROLL
-        for (unsigned i_sw = 0; i_sw < CONFIG_T::stride_width; i_sw++) {
-            #pragma HLS UNROLL
-            for (unsigned i_fh = 0; i_fh < CONFIG_T::trfilt_height; i_fh++) {
-                #pragma HLS UNROLL
-                for (unsigned i_fw = 0; i_fw < CONFIG_T::trfilt_width; i_fw++) {
-                    #pragma HLS UNROLL
-                    unsigned filt_h_ind = i_sh + (CONFIG_T::trfilt_height-i_fh-1)*CONFIG_T::stride_height;
-                    unsigned filt_w_ind = i_sw + (CONFIG_T::trfilt_width-i_fw-1)*CONFIG_T::stride_width;
-                    for (unsigned i_nf = 0; i_nf < CONFIG_T::n_filt; i_nf++) {
-                        #pragma HLS UNROLL
-                        for (unsigned i_nc = 0; i_nc < CONFIG_T::n_chan; i_nc++) {
-                            #pragma HLS UNROLL
-                            if (filt_h_ind < CONFIG_T::filt_height && filt_w_ind < CONFIG_T::filt_width) {
-                                trfilt_weights[i_sh][i_sw][
-                                    i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_height * CONFIG_T::trfilt_width + 
-                                    i_fh * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
-                                    i_fw * CONFIG_T::n_chan + i_nc
-                                ]= weights[
-                                    i_nf * CONFIG_T::n_chan * CONFIG_T::filt_height * CONFIG_T::filt_width + 
-                                    filt_h_ind * CONFIG_T::n_chan * CONFIG_T::filt_width + 
-                                    filt_w_ind * CONFIG_T::n_chan + i_nc
-                                ];
-                            }
-                            else {
-                                trfilt_weights[i_sh][i_sw][
-                                    i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_height * CONFIG_T::trfilt_width + 
-                                    i_fh * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
-                                    i_fw * CONFIG_T::n_chan + i_nc
-                                ] = 0;
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
-
 template<class data_T, class res_T, typename CONFIG_T>
 void compute_output_buffer_tr_2d(
     const data_T& in_elem,
     ap_shift_reg<typename data_T::value_type, CONFIG_T::in_width> line_buffer[MAX(CONFIG_T::trfilt_height-1, 1)][CONFIG_T::n_chan],
     hls::stream<res_T> &res_stream,
-    typename CONFIG_T::weight_t weights[CONFIG_T::kernel_size * CONFIG_T::n_chan * CONFIG_T::n_filt],
+    typename CONFIG_T::weight_t weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
+        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
+    ],
     typename CONFIG_T::bias_t biases[CONFIG_T::n_filt]
 )
 {
@@ -138,12 +88,6 @@ void compute_output_buffer_tr_2d(
     static typename data_T::value_type kernel_data[CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_chan];
     #pragma HLS ARRAY_PARTITION variable=kernel_data complete
 
-    static typename CONFIG_T::weight_t trfilt_weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
-        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
-    ];
-
-    load_trfilt_weights<CONFIG_T>(trfilt_weights, weights);
-
     typename res_T::value_type res_out[CONFIG_T::n_filt];
     #pragma HLS ARRAY_PARTITION variable=res_out complete dim = 0
 
@@ -168,11 +112,11 @@ void compute_output_buffer_tr_2d(
 
             if (CONFIG_T::strategy == nnet::latency) {
                 dense_latency<typename data_T::value_type, typename res_T::value_type, typename CONFIG_T::mult_config>(
-                    kernel_data, res_out, trfilt_weights[h_idx][w_idx], biases
+                    kernel_data, res_out, weights[h_idx][w_idx], biases
                 );
             } else {
                 dense_resource<typename data_T::value_type, typename res_T::value_type, typename CONFIG_T::mult_config>(
-                    kernel_data, res_out, trfilt_weights[h_idx][w_idx], biases
+                    kernel_data, res_out, weights[h_idx][w_idx], biases
                 );
             }
 
@@ -182,10 +126,7 @@ void compute_output_buffer_tr_2d(
                     (pX*CONFIG_T::stride_width+w_idx)*CONFIG_T::stride_height*CONFIG_T::n_filt + 
                     h_idx*CONFIG_T::n_filt + i_ic
                 ] = res_out[i_ic];
-                // res_pack[i_ic] = res_out[i_ic];
             }
-            // res_stream.write(res_pack);
-
         }
     }
 
@@ -226,7 +167,9 @@ template<class data_T, class res_T, typename CONFIG_T>
 void conv_2d_transpose_buffer_cl(
     hls::stream<data_T> &data,
     hls::stream<res_T>  &res,
-    typename CONFIG_T::weight_t weights[CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan * CONFIG_T::n_filt],
+    typename CONFIG_T::weight_t weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
+        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
+    ],
     typename CONFIG_T::bias_t   biases[CONFIG_T::n_filt]
 )
 {
@@ -248,7 +191,9 @@ template<class data_T, class res_T, typename CONFIG_T>
 void conv_2d_transpose_cl(
     hls::stream<data_T> &data,
     hls::stream<res_T> &res,
-    typename CONFIG_T::weight_t weights[CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan * CONFIG_T::n_filt],
+    typename CONFIG_T::weight_t weights[CONFIG_T::stride_height][CONFIG_T::stride_width][
+        CONFIG_T::trfilt_height * CONFIG_T::trfilt_width * CONFIG_T::n_filt * CONFIG_T::n_chan
+    ],
     typename CONFIG_T::bias_t  biases[CONFIG_T::n_filt]
 )
 {

hls4ml/templates/vivado/nnet_utils/nnet_helpers.h

@@ -103,6 +103,47 @@ void load_weights_from_txt(T w[DIM_1][DIM_2], const char* fname) {
     }
 }
 
+template<class T, size_t DIM_1, size_t DIM_2, size_t DIM_3>
+void load_weights_from_txt(T w[DIM_1][DIM_2][DIM_3], const char* fname) {
+
+    std::string full_path = std::string(WEIGHTS_DIR) + "/" + std::string(fname);
+    std::ifstream infile(full_path.c_str(), std::ios::binary);
+
+    if (infile.fail()) {
+        std::cerr << "ERROR: file " << std::string(fname) << " does not exist" << std::endl;
+        exit(1);
+    }
+
+    std::string line;
+    if (std::getline(infile, line)) {
+        std::istringstream iss(line);
+        std::string token;
+
+        size_t i = 0;
+        size_t j = 0;
+        size_t k = 0;
+        size_t tot = 0;
+        while(std::getline(iss, token, ',')) {
+            std::istringstream(token) >> w[i][j][k];
+            k++;
+            if (k == DIM_3) {
+                k = 0;
+                j++;
+                if (j == DIM_2) {
+                    j = 0;
+                    i++;
+                }
+            }
+            tot++;
+        }
+
+        if (DIM_1*DIM_2*DIM_3 != tot) {
+            std::cerr << "ERROR: Expected " << DIM_1*DIM_2*DIM_3 << " values";
+            std::cerr << " but read only " << tot << " values" << std::endl;
+        }
+    }
+}
+
 template<class T, size_t SIZE>
 void load_compressed_weights_from_txt(T *w, const char* fname) {
 

hls4ml/writer/vivado_writer.py

@@ -149,6 +149,8 @@ def write_project_cpp(self, model):
                             dim_info = w.data_length
                             if w.keep_dims == 1:
                                 dim_info = '{}, {}'.format(w.shape[0], w.data_length//w.shape[0])
+                            if w.keep_dims == 2:
+                                dim_info = '{}, {}, {}'.format(w.shape[0], w.shape[1], w.data_length//(w.shape[0]*w.shape[1]))
                             newline += indent + '    nnet::load_weights_from_txt<{}, {}>({}, "{}.txt");\n'.format(w.type.name, dim_info, w.name, w.name)
 
             #Add input/output type