move transposing of weight matrix to resource_strategy for transpose layers

Author: Jonathan-Shoemaker

hls4ml/backends/vivado/passes/resource_strategy.py

@@ -1,13 +1,14 @@
 import numpy as np
 
 from hls4ml.model.optimizer import OptimizerPass
-from hls4ml.model.layers import Conv1D, Conv2D, Dense, SeparableConv1D, SeparableConv2D, LSTM, GRU
+from hls4ml.model.layers import Conv1D, Conv2D, Conv1DTranspose, Conv2DTranspose, Dense, SeparableConv1D, SeparableConv2D, LSTM, GRU
 
 class ApplyResourceStrategy(OptimizerPass):
     ''' Transposes the weights to use the dense_resource matrix multiply routine '''
     def match(self, node):
 
-        node_matches = isinstance(node, (Dense, Conv1D, SeparableConv1D, Conv2D, SeparableConv2D, LSTM, GRU))
+        node_matches = isinstance(node, (Dense, Conv1D, SeparableConv1D, Conv2D, Conv1DTranspose, Conv2DTranspose, SeparableConv2D, LSTM, GRU))
+
         is_resource_strategy = node.get_attr('strategy', '').lower() == 'resource'
         already_transformed = node.get_attr('_weights_transposed', False) == True
 
@@ -18,11 +19,15 @@ def transform(self, model, node):
             node.weights['weight'].data = np.transpose(node.weights['weight'].data)
         elif isinstance(node, Conv1D):
             node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[2, 0, 1]) #(W,C,F) => (F,W,C)
+        elif isinstance(node, Conv1DTranspose):
+            node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[1, 0, 2]) #(W,F,C) => (F,W,C)
         elif isinstance(node, SeparableConv1D):
             node.weights['depthwise'].data = np.transpose(node.weights['depthwise'].data, axes=[2, 0, 1]) #(W,C,F) => (F,W,C)
             node.weights['pointwise'].data = np.transpose(node.weights['pointwise'].data, axes=[2, 0, 1]) #(W,C,F) => (F,W,C)
         elif isinstance(node, Conv2D):
             node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[3, 0, 1, 2]) #(H,W,C,F) => (F,H,W,C)
+        elif isinstance(node, Conv2DTranspose):
+            node.weights['weight'].data = np.transpose(node.weights['weight'].data, axes=[2, 0, 1, 3]) #(H,W,F,C) => (F,H,W,C)
         elif isinstance(node, SeparableConv2D):
             node.weights['depthwise'].data = np.transpose(node.weights['depthwise'].data, axes=[3, 0, 1, 2]) #(H,W,C,F) => (F,H,W,C)
             node.weights['pointwise'].data = np.transpose(node.weights['pointwise'].data, axes=[3, 0, 1, 2]) #(H,W,C,F) => (F,H,W,C)

hls4ml/templates/vivado/nnet_utils/nnet_conv1dtranspose_resource.h

@@ -51,12 +51,12 @@ void conv_1d_transpose_resource_cl(
                         trfilt_weights[
                             i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_width + i_fw  * CONFIG_T::n_chan + i_nc
                         ][i_sw] = weights[
-                            filt_ind * CONFIG_T::n_filt * CONFIG_T::n_chan + i_nf * CONFIG_T::n_chan + i_nc
+                            i_nf * CONFIG_T::n_chan * CONFIG_T::filt_width + filt_ind * CONFIG_T::n_chan + i_nc
                         ];
                     }
                     else {
                         trfilt_weights[
-                            i_fw * CONFIG_T::n_chan + i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_width + i_nc
+                            i_nf * CONFIG_T::n_chan * CONFIG_T::trfilt_width + i_fw * CONFIG_T::n_chan + i_nc
                         ][i_sw] = 0;
                     }
                 }

hls4ml/templates/vivado/nnet_utils/nnet_conv1dtranspose_stream.h

@@ -33,7 +33,6 @@ void weights_trim(
             #pragma HLS UNROLL 
             #pragma HLS PIPELINE
             for (int chan_ind = 0; chan_ind < CONFIG_T::n_chan; chan_ind++) {
-                // #pragma HLS LOOP_FLATTEN
                 #pragma HLS UNROLL 
                 #pragma HLS PIPELINE
                 if (row_indices[step] >= CONFIG_T::filt_width) {
@@ -42,14 +41,12 @@ void weights_trim(
                 } else {
                     row_weights[filt_ind * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
                         step * CONFIG_T::n_chan + chan_ind] = 
-                        weights[row_indices[step] * CONFIG_T::n_filt * CONFIG_T::n_chan + 
-                            filt_ind * CONFIG_T::n_chan + chan_ind];
+                        weights[filt_ind * CONFIG_T::filt_width * CONFIG_T::n_chan +
+                            row_indices[step] * CONFIG_T::n_chan + chan_ind];
                 }
             }
         }
     }
-    //try to split if else into two loops
-    //pre-compute
 }
 
 template <class data_T, typename CONFIG_T>
@@ -116,36 +113,27 @@ void compute_output_buffer_tr_1d(
 
     //always do stride number of multiplications
     StrideLoop: for (int idx = 0; idx < CONFIG_T::stride_width; idx++) {
-        // #pragma HLS DATAFLOW
-        // #pragma HLS PIPELINE
-        // #pragma HLS INLINE region
         //load in the weights for this multiplication
-        WeightsRegion: {
-            weights_trim<CONFIG_T>(
-                weights, row_weights, weight_start
-            );
+        weights_trim<CONFIG_T>(
+            weights, row_weights, weight_start
+        );
+
+        // Dense multiply
+        if (CONFIG_T::strategy == nnet::latency) {
+            dense_latency<typename data_T::value_type, typename res_T::value_type, typename CONFIG_T::mult_config>(
+                kernel_data, res_out, row_weights, biases);
+        } else {
+            dense_resource<typename data_T::value_type, typename res_T::value_type, typename CONFIG_T::mult_config>(
+                kernel_data, res_out, row_weights, biases);
         }
 
-        MultRegion: {
-            // Dense multiply
-            if (CONFIG_T::strategy == nnet::latency) {
-                dense_latency<typename data_T::value_type, typename res_T::value_type, typename CONFIG_T::mult_config>(
-                    kernel_data, res_out, row_weights, biases);
-            } else {
-                dense_resource<typename data_T::value_type, typename res_T::value_type, typename CONFIG_T::mult_config>(
-                    kernel_data, res_out, row_weights, biases);
-            }
-        }
-
-        PackRegion: {
-            // Pack output
-            if (oX >= CONFIG_T::pad_left && oX < CONFIG_T::pad_left + CONFIG_T::out_width) {
-                CastLoop: for (unsigned i_ic = 0; i_ic < CONFIG_T::n_filt; i_ic++) {
-                    #pragma HLS UNROLL
-                    res_pack[i_ic] = res_out[i_ic];
-                }
-                res_stream.write(res_pack);
+        // Pack output
+        if (oX >= CONFIG_T::pad_left && oX < CONFIG_T::pad_left + CONFIG_T::out_width) {
+            CastLoop: for (unsigned i_ic = 0; i_ic < CONFIG_T::n_filt; i_ic++) {
+                #pragma HLS UNROLL
+                res_pack[i_ic] = res_out[i_ic];
             }
+            res_stream.write(res_pack);
         }
         // Write output to stream when output ready
         oX++;

hls4ml/templates/vivado/nnet_utils/nnet_conv2dtranspose_resource.h

@@ -57,9 +57,9 @@ void conv_2d_transpose_resource_cl(
                                     i_fh * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
                                     i_fw * CONFIG_T::n_chan + i_nc
                                 ][i_sh][i_sw] = weights[
-                                    filt_h_ind * CONFIG_T::filt_width * CONFIG_T::n_filt * CONFIG_T::n_chan + 
-                                    filt_w_ind * CONFIG_T::n_filt * CONFIG_T::n_chan + 
-                                    i_nf * CONFIG_T::n_chan + i_nc
+                                    i_nf * CONFIG_T::n_chan * CONFIG_T::filt_height * CONFIG_T::filt_width + 
+                                    filt_h_ind * CONFIG_T::n_chan * CONFIG_T::filt_width + 
+                                    filt_w_ind * CONFIG_T::n_chan + i_nc
                                 ];
                             }
                             else {

hls4ml/templates/vivado/nnet_utils/nnet_conv2dtranspose_stream.h

@@ -112,9 +112,9 @@ void load_tr_kern_weights(
                             y_step * CONFIG_T::trfilt_width * CONFIG_T::n_chan + 
                             x_step * CONFIG_T::n_chan + chan_ind
                         ] = weights[
-                            y_indices[y_step] * CONFIG_T::filt_width * CONFIG_T::n_filt * CONFIG_T::n_chan +
-                            x_indices[x_step] * CONFIG_T::n_filt * CONFIG_T::n_chan + 
-                            filt_ind * CONFIG_T::n_chan + chan_ind
+                            filt_ind * CONFIG_T::filt_height * CONFIG_T::filt_width * CONFIG_T::n_chan + 
+                            y_indices[y_step] * CONFIG_T::filt_width * CONFIG_T::n_chan +
+                            x_indices[x_step] * CONFIG_T::n_chan + chan_ind
                         ];
                     }
                 }