pytest based synthesis tests

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "hls4ml/templates/catapult/ac_math"]
 	path = hls4ml/templates/catapult/ac_math
 	url = https://github.com/hlslibs/ac_math.git
+[submodule "test/pytest/baselines"]
+	path = test/pytest/baselines
+	url = https://github.com/marco66colombo/baselines.git

test/pytest/ci-template.yml

@@ -9,11 +9,21 @@
     - source /opt/intel/oneapi/setvars.sh --force
     - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml
     - git submodule update --init --recursive hls4ml/templates/catapult/
+    - git submodule update --init --recursive test/pytest/
     - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi
     - pip install .[testing,sr,optimization]
+    - export RUN_SYNTHESIS=true
+    - export TOOL_VERSION=2020.1
+    - mkdir -p cmd_vivado_${TOOL_VERSION}
+    - echo '#!/bin/bash' > cmd_vivado_${TOOL_VERSION}/vivado_hls
+    - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/ vivado_hls \"\$@\"" >> cmd_vivado_${TOOL_VERSION}/vivado_hls
+    - chmod +x cmd_vivado_${TOOL_VERSION}/vivado_hls
+    - export PATH=$PWD/cmd_vivado_${TOOL_VERSION}:$PATH
+    - which vivado_hls
+    - vivado_hls -version
   script:
     - cd test/pytest
-    - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed
+    - pytest -s $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed
   artifacts:
     when: always
     reports:
@@ -24,3 +34,4 @@
         path: test/pytest/coverage.xml
     paths:
       - test/pytest/hls4mlprj*.tar.gz
+      - test/pytest/synthesis_report_*.json

test/pytest/conftest.py

@@ -0,0 +1,35 @@
+import os
+
+import pytest
+
+
+def str_to_bool(val):
+    return str(val).lower() in ("1", "true")
+
+
+@pytest.fixture(scope="module")
+def synthesis_config():
+    """
+    Fixture that provides synthesis configuration for tests.
+
+    It gathers:
+    - Whether synthesis should be run (from the RUN_SYNTHESIS env var)
+    - Tool versions for each supported backend (from env vars)
+    - Build arguments specific to each backend toolchain
+
+    """
+    return {
+        "run_synthesis": str_to_bool(os.getenv("RUN_SYNTHESIS", "false")),
+        "tools_version": {
+            "Vivado": os.getenv("VIVADO_VERSION", "2020.1"),
+            "Vitis": os.getenv("VITIS_VERSION", "2020.1"),
+            "Quartus": os.getenv("QUARTUS_VERSION", "latest"),
+            "oneAPI": os.getenv("ONEAPI_VERSION", "2025.0.1"),
+        },
+        "build_args": {
+            "Vivado": {"csim": False, "synth": True, "export": False},
+            "Vitis": {"csim": False, "synth": True, "export": False},
+            "Quartus": {"synth": True, "fpgasynth": False},
+            "oneAPI": {"build_type": "report", "run": False},
+        },
+    }

test/pytest/generate_ci_yaml.py

@@ -1,84 +1,150 @@
-import itertools
-import os
-from pathlib import Path
-
 import yaml
 
 '''
-Create a Gitlab CI yml file with a separate entry for each test_* file
-in the pytests directory to parallelise the CI jobs.
+Minimal GitLab CI yml generator for testing purposes.
+Generates a single job entry for test_keras_api.py::test_activations.
 '''
 
-
 template = """
-pytest.{}:
+pytest.keras_api_part2:
   extends: .pytest
   variables:
-    PYTESTFILE: {}
-    EXAMPLEMODEL: {}
+    PYTESTFILE: test_keras_api.py::test_conv1d
+    EXAMPLEMODEL: 0
 """
 
 
-n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4))
-
-# Blacklisted tests will be skipped
-BLACKLIST = {'test_reduction'}
-
-# Long-running tests will not be bundled with other tests
-LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'}
-
-
-def path_to_name(test_path):
-    path = Path(test_path)
-    name = path.stem.replace('test_', '')
-    return name
-
-
-def batched(iterable, chunk_size):
-    iterator = iter(iterable)
-    while chunk := tuple(itertools.islice(iterator, chunk_size)):
-        yield chunk
-
-
-def uses_example_model(test_filename):
-    with open(test_filename) as f:
-        content = f.read()
-        return 'example-models' in content
-
-
-def generate_test_yaml(test_root='.'):
-    test_root = Path(test_root)
-    test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem not in (BLACKLIST | LONGLIST)]
-    need_example_models = [uses_example_model(path) for path in test_paths]
-
-    idxs = list(range(len(need_example_models)))
-    idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}')
-
-    yml = None
-    for batch_idxs in batched(idxs, n_test_files_per_yml):
-        batch_paths: list[Path] = [test_paths[i] for i in batch_idxs]
-        names = [path_to_name(path) for path in batch_paths]
-        name = '+'.join(names)
-        test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths])
-        batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs]))
-        diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model))
-        if yml is None:
-            yml = diff_yml
-        else:
-            yml.update(diff_yml)
-
-    test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST]
-    for path in test_paths:
-        name = path.stem.replace('test_', '')
-        test_file = str(path.relative_to(test_root))
-        needs_examples = uses_example_model(path)
-        diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples)))
-        yml.update(diff_yml)
-
-    return yml
+def generate_test_yaml():
+    return yaml.safe_load(template)
 
 
 if __name__ == '__main__':
-    yml = generate_test_yaml(Path(__file__).parent)
+    yml = generate_test_yaml()
     with open('pytests.yml', 'w') as yamlfile:
         yaml.safe_dump(yml, yamlfile)
+
+# import ast
+# import itertools
+# import os
+# from pathlib import Path
+
+# import yaml
+
+# '''
+# Create a Gitlab CI yml file with a separate entry for each test_* file
+# in the pytests directory to parallelise the CI jobs.
+# '''
+
+
+# template = """
+# pytest.{}:
+#   extends: .pytest
+#   variables:
+#     PYTESTFILE: {}
+#     EXAMPLEMODEL: {}
+# """
+
+
+# n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4))
+
+# # Blacklisted tests will be skipped
+# BLACKLIST = {'test_reduction'}
+
+# # Long-running tests will not be bundled with other tests
+# LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'}
+
+# # Test files to split by individual test cases (stem only, no .py)
+# # Value = chunk size per CI job
+# SPLIT_BY_TEST_CASE = {
+#     'test_keras_api': 1,
+# }
+
+
+# def collect_test_functions_from_ast(test_file):
+#     """Collect all test function names using AST parsing (no imports)."""
+#     with open(test_file, encoding='utf-8') as f:
+#         tree = ast.parse(f.read(), filename=str(test_file))
+
+#     test_funcs = []
+#     for node in tree.body:
+#         if isinstance(node, ast.FunctionDef) and node.name.startswith("test"):
+#             test_funcs.append(f"{test_file}::{node.name}")
+#     return test_funcs
+
+
+# def batched(iterable, batch_size):
+#     it = iter(iterable)
+#     while batch := list(itertools.islice(it, batch_size)):
+#         yield batch
+
+
+# def path_to_name(test_path):
+#     path = Path(test_path)
+#     name = path.stem.replace('test_', '')
+#     return name
+
+
+# def uses_example_model(test_filename):
+#     with open(test_filename) as f:
+#         content = f.read()
+#         return 'example-models' in content
+
+
+# def generate_test_yaml(test_root='.'):
+#     test_root = Path(test_root)
+#     test_paths = [
+#         path
+#         for path in test_root.glob('**/test_*.py')
+#         if path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys()))
+#     ]
+#     need_example_models = [uses_example_model(path) for path in test_paths]
+
+#     idxs = list(range(len(need_example_models)))
+#     idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}')
+
+#     yml = None
+#     for batch_idxs in batched(idxs, n_test_files_per_yml):
+#         batch_paths: list[Path] = [test_paths[i] for i in batch_idxs]
+#         names = [path_to_name(path) for path in batch_paths]
+#         name = '+'.join(names)
+#         test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths])
+#         batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs]))
+#         diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model))
+#         if yml is None:
+#             yml = diff_yml
+#         else:
+#             yml.update(diff_yml)
+
+#     test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST]
+#     for path in test_paths:
+#         name = path.stem.replace('test_', '')
+#         test_file = str(path.relative_to(test_root))
+#         needs_examples = uses_example_model(path)
+#         diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples)))
+#         yml.update(diff_yml)
+
+#     test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE]
+#     for path in test_paths:
+#         stem = path.stem
+#         name_base = stem.replace('test_', '')
+#         test_file = str(path.relative_to(test_root))
+#         test_ids = collect_test_functions_from_ast(test_file)
+#         chunk_size = SPLIT_BY_TEST_CASE[stem]
+#         needs_examples = uses_example_model(path)
+
+#         for i, batch in enumerate(batched(test_ids, chunk_size)):
+#             job_name = f"{name_base}_part{i}"
+#             test_file_args = " ".join(batch).strip().replace("\n", " ")
+#             diff_yml = yaml.safe_load(template.format(job_name, test_file_args, int(needs_examples)))
+#             if yml is None:
+#                 yml = diff_yml
+#             else:
+#                 yml.update(diff_yml)
+
+#     return yml
+
+
+# if __name__ == '__main__':
+#     yml = generate_test_yaml(Path(__file__).parent)
+#     with open('pytests.yml', 'w') as yamlfile:
+#         yaml.safe_dump(yml, yamlfile)