Merge branch 'main' into GRUv1

Author: JanFSchulte

hls4ml/converters/pytorch_to_hls.py

@@ -95,6 +95,7 @@ def decorator(function):
     'avg_pool1d': 'AvgPool1d',
     'avg_pool2d': 'AvgPool2d',
     'flatten': 'Flatten',
+    'view': 'View',
 }
 
 

test/pytest/generate_ci_yaml.py

@@ -1,6 +1,6 @@
-import glob
 import itertools
 import os
+from pathlib import Path
 
 import yaml
 
@@ -9,6 +9,7 @@
 in the pytests directory to parallelise the CI jobs.
 '''
 
+
 template = """
 pytest.{}:
   extends: .pytest
@@ -19,6 +20,14 @@
 
 n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4))
 
+BLACKLIST = {'test_reduction'}
+
+
+def path_to_name(test_path):
+    path = Path(test_path)
+    name = path.stem.replace('test_', '')
+    return name
+
 
 def batched(iterable, chunk_size):
     iterator = iter(iterable)
@@ -32,41 +41,32 @@ def uses_example_model(test_filename):
         return 'example-models' in content
 
 
-yml = None
-tests = glob.glob('test_*.py')
-for test_batch in batched(tests, n_test_files_per_yml):
-    name = '+'.join([test.replace('test_', '').replace('.py', '') for test in test_batch])
-    test_files = ' '.join(list(test_batch))
-    uses_example_models = int(any([uses_example_model(test) for test in test_batch]))
-
-    new_yml = yaml.safe_load(template.format(name, test_files, uses_example_models))
-    if yml is None:
-        yml = new_yml
-    else:
-        yml.update(new_yml)
-
-# hls4ml Optimization API
-tests = glob.glob('test_optimization/test_*.py')
-for test in tests:
-    name = test.replace('test_optimization/', '').replace('test_', '').replace('.py', '')
-    new_yml = yaml.safe_load(template.format(name, f'test_optimization/test_{name}.py', int(uses_example_model(test))))
-    if yml is None:
-        yml = new_yml
-    else:
-        yml.update(new_yml)
-
-tests = glob.glob('test_optimization/test_keras/test_*.py')
-for test in tests:
-    # For now, skip Keras Surgeon [conflicting versions]
-    if 'test_reduction' not in test:
-        name = test.replace('test_optimization/test_keras/', '').replace('test_', '').replace('.py', '')
-        new_yml = yaml.safe_load(
-            template.format(name, f'test_optimization/test_keras/test_{name}.py', int(uses_example_model(test)))
-        )
+def generate_test_yaml(test_root='.'):
+    test_root = Path(test_root)
+    test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem not in BLACKLIST]
+    for path in test_paths:
+        print(path.name)
+    need_example_models = [uses_example_model(path) for path in test_paths]
+
+    idxs = list(range(len(need_example_models)))
+    idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}')
+
+    yml = None
+    for batch_idxs in batched(idxs, n_test_files_per_yml):
+        batch_paths: list[Path] = [test_paths[i] for i in batch_idxs]
+        names = [path_to_name(path) for path in batch_paths]
+        name = '+'.join(names)
+        test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths])
+        batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs]))
+        diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model))
         if yml is None:
-            yml = new_yml
+            yml = diff_yml
         else:
-            yml.update(new_yml)
+            yml.update(diff_yml)
+    return yml
+
 
-yamlfile = open('pytests.yml', 'w')
-yaml.safe_dump(yml, yamlfile)
+if __name__ == '__main__':
+    yml = generate_test_yaml(Path(__file__).parent)
+    with open('pytests.yml', 'w') as yamlfile:
+        yaml.safe_dump(yml, yamlfile)

test/pytest/test_pytorch_api.py

@@ -808,3 +808,65 @@ def forward(self, x):
     hls_prediction = hls_model.predict(hls_input).flatten()
 
     np.testing.assert_allclose(hls_prediction, pytorch_prediction, rtol=0, atol=5e-2)
+
+
+@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus'])
+@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream'])
+def test_view(backend, io_type):
+
+    class TestModel(nn.Module):
+        def __init__(self, n_in, n_out, size_in):
+            super().__init__()
+            self.view_mult = n_out * size_in
+
+            self.conv1 = nn.Conv1d(
+                n_in,
+                n_out,
+                kernel_size=3,
+                padding=1,
+                bias=False,
+            )
+
+        def forward(self, x):
+            z = self.conv1(x)
+            z = z.view(-1, self.view_mult)
+            return z
+
+    n_in = 2
+    n_out = 4
+    size_in = 128
+    n_batch = 100
+
+    model = TestModel(n_in, n_out, size_in)
+    model = model.to(memory_format=torch.channels_last)
+    model.eval()
+
+    X_input = np.random.rand(n_batch, n_in, size_in)
+    pytorch_prediction = model(torch.Tensor(X_input)).detach().numpy()
+
+    # X_input is channels last
+    X_input = np.ascontiguousarray(X_input.transpose(0, 2, 1))
+    config = config_from_pytorch_model(model, inputs_channel_last=True, transpose_outputs=False)
+
+    output_dir = str(test_root_path / f'hls4mlprj_pytorch_view_{backend}_{io_type}')
+    hls_model = convert_from_pytorch_model(
+        model,
+        (None, n_in, size_in),
+        hls_config=config,
+        output_dir=output_dir,
+        backend=backend,
+        io_type=io_type,
+    )
+
+    hls_model.compile()
+
+    # reshape hls prediction to channels last, then transpose, then reshape
+    # to match .view
+    hls_prediction = np.reshape(
+        np.transpose(np.reshape(hls_model.predict(X_input), (n_batch, size_in, n_out)), (0, 2, 1)),
+        (n_batch, size_in * n_out),
+    )
+
+    rtol = 0
+    atol = 5.0e-2
+    np.testing.assert_allclose(hls_prediction, pytorch_prediction, rtol=rtol, atol=atol)