ConvTranspose2d: Allow pad=0, pad=2 (#111)

* ConvTranspose2d: Allow pad=0, pad=2
* Changes for AI87
* README: Small improvements
* Allow 32-bit output for intermediate layers (with warning)
* Optimize Conv1d simulation

Author: Robert Muchsel

.github/linters/.python-lint

@@ -7,7 +7,7 @@ ignored-classes = ModelProto
 max-line-length = 99
 [DESIGN]
 max-locals=100
-max-statements=1200
+max-statements=1500
 min-public-methods=1
 max-branches=150
 max-module-lines=5000

.pylintrc

@@ -7,7 +7,7 @@ ignored-classes = ModelProto
 max-line-length = 99
 [DESIGN]
 max-locals=100
-max-statements=1200
+max-statements=1500
 min-public-methods=1
 max-branches=150
 max-module-lines=5000

README.md

@@ -1,6 +1,6 @@
 # MAX78000 Model Training and Synthesis
 
-_February 19, 2021_
+_March 2, 2021_
 
 The Maxim Integrated AI project is comprised of four repositories:
 
@@ -204,7 +204,7 @@ Windows/MS-DOS is not supported for training networks at this time. *This includ
 
 ### Upstream Code
 
-Change to the project root and run the following commands. Use your GitHub credentials when prompted.
+Change to the project root and run the following commands. Use your GitHub credentials if prompted.
 
 ```shell
 $ cd $AI_PROJECT_ROOT
@@ -318,6 +318,10 @@ $ source bin/activate
 
 Branches and updates for `ai8x-synthesis` are handled similarly to the [`ai8x-training`](#Repository Branches) project.
 
+**Installation is now Complete**
+
+With the installation of Training and Synthesis projects completed it is important to remember to activate the proper Python virtual environment when switching between projects. If scripts begin failing in a previously working environment, the cause might be that the incorrect virtual environment is active or that no virtual environment has been activated.
+
 ### Embedded Software Development Kit (SDK)
 
 The MAX78000 SDK is a git submodule of ai8x-synthesis. It is checked out automatically to a version compatible with the project into the folder `sdk`.
@@ -688,7 +692,7 @@ The MAX78000 hardware does not support arbitrary network parameters. Specificall
 * `ConvTranspose2d`:
 
   * Kernel sizes must be 3×3.
-  * Padding must be 1 *(Note: hardware supports additional padding modes, but there is no direct equivalent in PyToch)*.
+  * Padding can be 0, 1, or 2.
   * Stride is fixed to [2, 2]. Output padding is fixed to 1.
 
 * A programmable layer-specific shift operator is available at the output of a convolution, see [`output_shift` (Optional)](#output_shift \(Optional\)).
@@ -1456,13 +1460,9 @@ Example:
 
 ##### `kernel_size` (Optional)
 
-2D convolutions:
-
-​	This key must be `3x3` (the default) or `1x1`.
-
-1D convolutions:
-
-​	This key must be `1` through `9`.
+* For `Conv2D`, this key must be `3x3` (the default) or `1x1`.
+* For `Conv1D`, this key must be `1` through `9`.
+* For `ConvTranspose2D`, this key must be `3x3` (the default).
 
 Example:
 	`kernel_size: 1x1`
@@ -1477,6 +1477,7 @@ This key must be `1` or `[1, 1]`.
 
 * For `Conv2d`, this value can be `0`, `1` (the default), or `2`.
 * For `Conv1d`, the value can be `0`, `1`, `2`, or `3` (the default).
+* For `ConvTranspose2d`, this value can be `0`, `1` (the default), or `2`. *Note that the value follows PyTorch conventions and effectively adds* `(kernel_size – 1) – pad` *amount of zero padding to both sizes of the input, so “0” adds 2 zeros each and “2” adds no padding.*
 * For `Passthrough`, this value must be `0` (the default).
 
 ##### `max_pool` (Optional)

README.pdf

@@ -29,6 +29,7 @@ def load(
         verbose=False,
         no_bias=None,
         conv_groups=None,
+        bypass=None,
 ):
     """
     Load weights and biases from `checkpoint_file`. If `arch` is not None and does not match
@@ -76,7 +77,7 @@ def load(
 
     for _, k in enumerate(checkpoint_state.keys()):
         # Skip over non-weight layers
-        while seq < len(operator) and operator[seq] == opn.NONE:
+        while seq < len(operator) and (operator[seq] == opn.NONE or bypass[seq]):
             seq += 1
 
         operation, parameter = k.rsplit(sep='.', maxsplit=1)

izer/checkpoint.py

@@ -62,13 +62,13 @@ def create_net(  # pylint: disable=too-many-arguments,too-many-locals,too-many-b
         legacy_test=False,
 ):
     """
-    Create the CMSIS NN network.
+    Create the CMSIS-NN network.
     """
     wprint('CMSIS-NN code generation is unsupported.')
 
     if output_width[-1] != 8:
-        wprint('CMSIS network generator does not currently support `output_width` that is not 8. '
-               'Forcing to 8 bit.')  # FIXME: Support 32-bit output
+        wprint('CMSIS-NN network generator does not currently support `output_width` that '
+               'is not 8. Forcing to 8 bit.')  # FIXME: Support 32-bit output
         output_width[-1] = 8
 
     input_dim_str = [None] * layers
@@ -83,7 +83,7 @@ def create_net(  # pylint: disable=too-many-arguments,too-many-locals,too-many-b
         if quantization[ll] is None:
             quantization[ll] = 8  # Set default
         elif quantization[ll] != 8:  # FIXME: Support quantization
-            eprint('CMSIS network generator does not currently support `quantization` != 8.')
+            eprint('CMSIS-NN network generator does not currently support `quantization` != 8.')
 
         if output_shift[ll] is None:
             output_shift[ll] = 0  # Set default

izer/cmsisnn.py

@@ -123,6 +123,8 @@ def get_parser():
                        help="start processing before first FIFO push (default: false)")
     group.add_argument('--slow-load', type=int, metavar='N', default=0,
                        help="slow down FIFO loads (default: 0)")
+    group.add_argument('--debug-new-streaming', action='store_true', default=True,
+                       help="modify streaming equation (default: false)")
 
     # RISC-V
     group = parser.add_argument_group('RISC-V')
@@ -155,7 +157,8 @@ def get_parser():
     group.add_argument('--log-pooling', action='store_true', default=False,
                        help="log unpooled and pooled data between layers in CSV format "
                             "(default: false)")
-    group.add_argument('--log-last-only', action='store_false', dest='verbose_all', default=True,
+    group.add_argument('--log-last-only', '--verbose-all',
+                       action='store_false', dest='verbose_all', default=True,
                        help="log data for last layer only (default: all layers)")
     group.add_argument('--log-filename', default='log.txt', metavar='S',
                        help="log file name (default: 'log.txt')")
@@ -200,6 +203,8 @@ def get_parser():
                        help="do not force `bias_group` to use an active group (default: false)")
     group.add_argument('--kernel-format', default='{0:4}', metavar='S',
                        help="print format for kernels (default: '{0:4}')")
+    group.add_argument('--debug-snoop', action='store_true', default=False,
+                       help="insert snoop register debug code (default: False)")
 
     # RTL sim
     group = parser.add_argument_group('RTL simulation')

izer/commandline.py

@@ -102,8 +102,8 @@ def conv2d(
         nweight[:, :, 0::dilation[0], 0::dilation[1]] = weight
         weight = nweight
 
-    h = (data.shape[1] - weight.shape[3] + 1) // stride[0]  # Resulting output height
-    w = (data.shape[2] - weight.shape[2] + 1) // stride[1]  # Resulting output width
+    h = (data.shape[1] - weight.shape[3]) // stride[0] + 1  # Resulting output height
+    w = (data.shape[2] - weight.shape[2]) // stride[1] + 1  # Resulting output width
 
     view = as_strided(data,
                       shape=(h, w, data.shape[0], weight.shape[2], weight.shape[3]),
@@ -177,57 +177,114 @@ def conv1d(
         bias,
         input_size,
         output_size,
-        out_channels,
         kernel_size,
         stride,
         pad,
         dilation,
+        fractional_stride=1,
+        output_pad=0,
         groups=1,
-        debug=False,
+        debug=False,  # pylint: disable=unused-argument
 ):
     """
     Compute a 1D convolution.
 
     Note that all PyTorch numbers are ordered (C, L)
     """
+    assert data.shape == tuple(input_size)
     in_channels = input_size[0]
+    out_channels = output_size[0]
 
     weight = weight.reshape(out_channels, input_size[0] // groups, -1)
     data = data.reshape(input_size[0], -1)
 
     output = np.full(shape=(output_size[0], output_size[1]),
                      fill_value=np.nan, dtype=np.int64)
 
-    # Compute 1D convolution
-    if debug:
-        debug_print('k,c,x,src_offs,wt_offs,weight,data,acc')
-    for k in range(out_channels):
-        out_offs = 0
-        for x in range(-pad, input_size[1] - dilation * (kernel_size - 1) + pad, stride):
-            val = np.int64(0)
-            for c in range(in_channels // groups):
-                dc = c if groups == 1 else c + k * (in_channels // groups)
-                for w in range(kernel_size):
-                    src_offs = x + w * dilation
-                    if 0 <= src_offs < input_size[1]:
-                        val += weight[k][c][w] * data[dc][src_offs]
-                        stats.true_macc += 1
-                        if debug:
-                            debug_print(
-                                f'{k},{c},{x},{src_offs},{w},{weight[k][c][w]},'
-                                f'{data[dc][src_offs]},{val}'
-                            )
-
-            if bias is not None:
-                val += bias[k]
-                if debug:
-                    debug_print(
-                        f'+bias {bias[k]} --> output[{k}][{out_offs}] = {val}',
-                    )
-            output[k][out_offs] = val
-            out_offs += 1
-
-    return output.reshape((output_size))
+    # Stretch data for fractionally-strided convolution
+    if fractional_stride > 1:
+        ndata = np.zeros((data.shape[0],
+                          data.shape[1] * fractional_stride - 1),
+                         dtype=data.dtype)
+        ndata[:, 0::fractional_stride] = data
+        data = ndata
+
+    # Create zero padding around data
+    if pad or output_pad:
+        data = np.pad(data, pad_width=((0, 0), (pad, pad + output_pad)),
+                      mode='constant', constant_values=0)
+
+    if dilation > 1:
+        # Stretch weights for dilation
+        nweight = np.zeros((weight.shape[0], weight.shape[1],
+                            (kernel_size - 1) * dilation + 1),
+                           dtype=weight.dtype)
+        nweight[:, :, 0::dilation] = weight
+        weight = nweight
+
+    ll = (data.shape[1] - weight.shape[2]) // stride + 1  # Resulting output length
+
+    view = as_strided(data,
+                      shape=(ll, data.shape[0], weight.shape[2]),
+                      strides=((data.strides[1] * stride,
+                                data.strides[0], data.strides[1])),
+                      writeable=False)
+
+    if groups > 1:
+        nweight = np.zeros((weight.shape[0], in_channels, weight.shape[2]),
+                           dtype=weight.dtype)
+        for i in range(weight.shape[0]):
+            for j in range(in_channels // groups):
+                nweight[i, i * (in_channels // groups) + j, :] = weight[i, j, :]
+        weight = nweight
+
+    output = np.tensordot(view, weight, axes=((1, 2), (1, 2))).transpose(1, 0)
+
+    # Apply bias
+    if bias is not None:
+        for k in range(out_channels):
+            output[k] += bias[k]
+
+    assert output.shape == tuple(output_size[:2]), \
+        f'Shape mismatch: NumPy result {output.shape} vs expected {tuple(output_size[:2])}.'
+
+    return output
+
+
+def convtranspose1d(
+        data,
+        weight,
+        bias,
+        input_size,
+        output_size,
+        kernel_size,
+        stride,
+        pad,
+        dilation,
+        fractional_stride,
+        output_pad,
+        groups=1,
+        debug=False,
+):
+    """
+    Compute a transposed 1D convolution.
+    """
+
+    return conv1d(
+        data,
+        weight,
+        bias,
+        input_size,
+        output_size,
+        kernel_size,
+        stride,
+        dilation * (kernel_size - 1) - pad,
+        dilation,
+        fractional_stride=fractional_stride,
+        output_pad=output_pad,
+        groups=groups,
+        debug=debug,
+    )
 
 
 def linear(