pytest based synthesis tests

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "hls4ml/templates/catapult/ac_math"]
 	path = hls4ml/templates/catapult/ac_math
 	url = https://github.com/hlslibs/ac_math.git
+[submodule "test/pytest/baselines"]
+	path = test/pytest/baselines
+	url = https://github.com/marco66colombo/baselines.git

test/pytest/ci-template.yml

@@ -11,6 +11,10 @@
     - git submodule update --init --recursive hls4ml/templates/catapult/
     - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi
     - pip install .[testing,sr,optimization]
+    - sudo yum install libtinfo.so.6 -y
+    - sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5
+    - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx
+    - source /opt/Xilinx/Vivado/2020.1/settings64.sh
   script:
     - cd test/pytest
     - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed
@@ -24,3 +28,4 @@
         path: test/pytest/coverage.xml
     paths:
       - test/pytest/hls4mlprj*.tar.gz
+      - test/pytest/synthesis_report_*.json

test/pytest/conftest.py

@@ -0,0 +1,35 @@
+import os
+
+import pytest
+
+
+def str_to_bool(val):
+    return str(val).lower() in ("1", "true")
+
+
+@pytest.fixture(scope="module")
+def synthesis_config():
+    """
+    Fixture that provides synthesis configuration for tests.
+
+    It gathers:
+    - Whether synthesis should be run (from the RUN_SYNTHESIS env var)
+    - Tool versions for each supported backend (from env vars)
+    - Build arguments specific to each backend toolchain
+
+    """
+    return {
+        "run_synthesis": str_to_bool(os.getenv("RUN_SYNTHESIS", "false")),
+        "tools_version": {
+            "Vivado": os.getenv("VIVADO_VERSION", "2020.1"),
+            "Vitis": os.getenv("VITIS_VERSION", "2020.1"),
+            "Quartus": os.getenv("QUARTUS_VERSION", "latest"),
+            "oneAPI": os.getenv("ONEAPI_VERSION", "latest"),
+        },
+        "build_args": {
+            "Vivado": {"csim": False, "synth": True, "export": False},
+            "Vitis": {"csim": False, "synth": True, "export": False},
+            "Quartus": {"synth": True, "fpgasynth": False},
+            "oneAPI": {"build_type": "fpga_emu", "run": False},
+        },
+    }

test/pytest/synthesis_helpers.py

@@ -0,0 +1,136 @@
+import json
+from pathlib import Path
+
+import pytest
+
+
+def get_baseline_path(baseline_file_name, backend, version):
+    """
+    Construct the full path to a baseline synthesis report file.
+
+    Args:
+        baseline_file_name (str): The name of the baseline report file.
+        backend (str): The backend used (e.g., 'Vivado', 'Vitis').
+        version (str): The tool version (e.g., '2020.1').
+
+    Returns:
+        Path: A pathlib.Path object pointing to the baseline file location.
+    """
+    return Path(__file__).parent / "baselines" / backend / version / baseline_file_name
+
+
+def save_report(data, filename):
+    """
+    Save synthesis data to a JSON file in the same directory as this script.
+
+    Args:
+        data (dict): The synthesis output data to be saved.
+        filename (str): The filename to write to (e.g., 'synthesis_report_test_x.json').
+
+    Raises:
+        OSError: If the file cannot be written.
+    """
+    out_path = Path(__file__).parent / filename
+    with open(out_path, "w") as fp:
+        json.dump(data, fp, indent=4)
+
+
+def get_tolerance(key):
+    """
+    Get the relative tolerance for a specific synthesis report field.
+
+    Args:
+        key (str): The key in the synthesis report to compare.
+
+    Returns:
+        float: The relative tolerance allowed for that key. Defaults to 1% (0.01).
+    """
+    tolerances = {
+        "EstimatedClockPeriod": 0.01,
+        "FF": 0.05,
+        "LUT": 0.10,
+        "BRAM_18K": 0.0,
+        "DSP": 0.0,
+        "URAM": 0.0,
+        "AvailableBRAM_18K": 0.0,
+        "AvailableDSP": 0.0,
+        "AvailableFF": 0.0,
+        "AvailableLUT": 0.0,
+        "AvailableURAM": 0.0,
+    }
+
+    default_tolerance = 0.01
+
+    return tolerances.get(key, default_tolerance)
+
+
+def compare_reports_with_tolerance(data, baseline):
+    """
+    Compare two synthesis reports using tolerances defined per key.
+
+    Args:
+        data (dict): The current synthesis report.
+        baseline (dict): The baseline synthesis report to compare against.
+    """
+    csrBaseline = baseline.get("CSynthesisReport")
+    csrData = data.get("CSynthesisReport")
+
+    for key, expected_value in csrBaseline.items():
+        actual_value = csrData.get(key)
+        tolerance = get_tolerance(key)
+
+        try:
+            # Convert to float for numerical comparison
+            expected_num = float(expected_value)
+            actual_num = float(actual_value)
+            assert actual_num == pytest.approx(
+                expected_num, rel=tolerance
+            ), f"{key}: expected {expected_num}, got {actual_num} (tolerance={tolerance*100}%)"
+        except ValueError:
+            # Exact match for non-numeric values
+            assert actual_value == expected_value, f"{key}: expected '{expected_value}', got '{actual_value}'"
+
+
+def test_synthesis(config, hls_model, baseline_file_name, backend):
+    """
+    Run HLS synthesis and compare the output with a stored baseline report.
+
+    If synthesis is disabled via the configuration (`run_synthesis=False`),
+    no synthesis is executed and the test silently returns.
+
+    Args:
+        config (dict): Test-wide synthesis configuration fixture.
+        hls_model (object): hls4ml model instance to build and synthesize.
+        baseline_file_name (str): The name of the baseline file for comparison.
+        backend (str): The synthesis backend used (e.g., 'Vivado', 'Vitis').
+    """
+    if not config.get("run_synthesis", False):
+        # TODO: should this info be printed or logged?
+        return
+
+    if backend == 'oneAPI':
+        pytest.skip('oneAPI backend not supported in synthesis tests.')
+
+    build_args = config["build_args"]
+
+    try:
+        data = hls_model.build(**build_args.get(backend, {}))
+    except Exception as e:
+        pytest.skip(f"hls_model.build failed: {e}")
+
+    save_report(data, f"synthesis_report_{baseline_file_name}")
+
+    assert data and {'CSynthesisReport'}.issubset(
+        data.keys()
+    ), "Synthesis failed: Missing expected keys in the synthesis report"
+
+    version = config["tools_version"].get(backend)
+    baseline_path = get_baseline_path(baseline_file_name, backend, version)
+
+    try:
+        with open(baseline_path) as fp:
+            baseline = json.load(fp)
+    except FileNotFoundError:
+        pytest.skip(f"Baseline file '{baseline_path}' not found.")
+
+    compare_reports_with_tolerance(data, baseline)

test/pytest/test_keras_api.py

@@ -4,6 +4,7 @@
 import numpy as np
 import pytest
 import tensorflow as tf
+from synthesis_helpers import test_synthesis
 from tensorflow.keras.layers import (
     ELU,
     Activation,
@@ -27,7 +28,7 @@
 
 @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI'])
 @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream'])
-def test_dense(backend, io_type):
+def test_dense(backend, io_type, synthesis_config):
     model = tf.keras.models.Sequential()
     model.add(
         Dense(
@@ -53,6 +54,7 @@ def test_dense(backend, io_type):
 
     config = hls4ml.utils.config_from_keras_model(model)
     output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}')
+    baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}.json'
 
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
@@ -74,6 +76,8 @@ def test_dense(backend, io_type):
     assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1]
     assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1]
 
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)
+
 
 # TODO: add ThresholdedReLU test when it can be made to pass
 # https://github.com/fastmachinelearning/hls4ml/issues/376
@@ -92,7 +96,7 @@ def test_dense(backend, io_type):
 # ThresholdedReLU(theta=1.0)])
 @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI'])
 @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream'])
-def test_activations(activation_function, backend, io_type):
+def test_activations(activation_function, backend, io_type, synthesis_config):
     model = tf.keras.models.Sequential()
     model.add(Dense(64, input_shape=(1,), name='Dense', kernel_initializer='lecun_uniform', kernel_regularizer=None))
     model.add(activation_function)
@@ -102,6 +106,8 @@ def test_activations(activation_function, backend, io_type):
     keras_prediction = model.predict(X_input)
     config = hls4ml.utils.config_from_keras_model(model)
     output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}')
+    baseline_file_name = f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}.json'
+
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
     )
@@ -114,6 +120,8 @@ def test_activations(activation_function, backend, io_type):
 
     assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__
 
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)
+
 
 padds_options = ['same', 'valid']
 
@@ -131,7 +139,7 @@ def test_activations(activation_function, backend, io_type):
     ],
 )
 @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream'])
-def test_conv1d(padds, backend, strategy, io_type):
+def test_conv1d(padds, backend, strategy, io_type, synthesis_config):
     model = tf.keras.models.Sequential()
     input_shape = (10, 128, 4)
     model.add(
@@ -156,6 +164,8 @@ def test_conv1d(padds, backend, strategy, io_type):
     config = hls4ml.utils.config_from_keras_model(model)
     config['Model']['Strategy'] = strategy
     output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}')
+    baseline_file_name = f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}.json'
+
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
     )
@@ -196,6 +206,8 @@ def test_conv1d(padds, backend, strategy, io_type):
             assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0
             assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0
 
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)
+
 
 chans_options = ['channels_last']
 padds_options = ['same', 'valid']
@@ -215,7 +227,7 @@ def test_conv1d(padds, backend, strategy, io_type):
     ],
 )
 @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream'])
-def test_conv2d(chans, padds, backend, strategy, io_type):
+def test_conv2d(chans, padds, backend, strategy, io_type, synthesis_config):
     model = tf.keras.models.Sequential()
     input_shape = (28, 28, 3)
     model.add(
@@ -238,6 +250,8 @@ def test_conv2d(chans, padds, backend, strategy, io_type):
     config = hls4ml.utils.config_from_keras_model(model)
     config['Model']['Strategy'] = strategy
     output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}')
+    baseline_file_name = f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}.json'
+
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
     )
@@ -316,11 +330,13 @@ def test_conv2d(chans, padds, backend, strategy, io_type):
         assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0
         assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0
 
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)
+
 
 # Currently only Vivado and Vitis is supported for io_stream.
 @pytest.mark.parametrize('backend', ['Vivado', 'Vitis'])
 @pytest.mark.parametrize('io_type', ['io_stream'])
-def test_depthwise2d(backend, io_type):
+def test_depthwise2d(backend, io_type, synthesis_config):
     '''
     Test proper handling of DepthwiseConv2D
     '''
@@ -334,6 +350,8 @@ def test_depthwise2d(backend, io_type):
         model, granularity='name', default_precision='fixed<32,12>', backend=backend
     )
     output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}')
+    baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}.json'
+
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
     )
@@ -344,11 +362,13 @@ def test_depthwise2d(backend, io_type):
 
     np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01)
 
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)
+
 
 # Currently only Vivado and Vitis is supported for io_stream.
 @pytest.mark.parametrize('backend', ['Vivado', 'Vitis'])
 @pytest.mark.parametrize('io_type', ['io_stream'])
-def test_depthwise1d(backend, io_type):
+def test_depthwise1d(backend, io_type, synthesis_config):
     '''
     Test proper handling of DepthwiseConv1D.
     '''
@@ -360,6 +380,8 @@ def test_depthwise1d(backend, io_type):
 
     config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend)
     output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}')
+    baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}.json'
+
     hls_model = hls4ml.converters.convert_from_keras_model(
         model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type
     )
@@ -370,6 +392,8 @@ def test_depthwise1d(backend, io_type):
 
     np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01)
 
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)
+
 
 pooling_layers = [MaxPooling1D, MaxPooling2D, AveragePooling1D, AveragePooling2D]
 
@@ -378,7 +402,7 @@ def test_depthwise1d(backend, io_type):
 @pytest.mark.parametrize('padds', padds_options)
 @pytest.mark.parametrize('chans', chans_options)
 @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI'])
-def test_pooling(pooling, padds, chans, backend):
+def test_pooling(pooling, padds, chans, backend, synthesis_config):
     assert '1D' in pooling.__name__ or '2D' in pooling.__name__
 
     input_shape = (18, 15, 3) if '2D' in pooling.__name__ else (121, 3)
@@ -392,6 +416,10 @@ def test_pooling(pooling, padds, chans, backend):
     output_dir = str(
         test_root_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}'
     )
+    baseline_file_name = (
+        f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}.json'
+    )
+
     hls_model = hls4ml.converters.convert_from_keras_model(
         keras_model, hls_config=hls_cfg, output_dir=output_dir, backend=backend
     )
@@ -495,3 +523,5 @@ def test_pooling(pooling, padds, chans, backend):
             assert hls_pool.attributes['n_out'] == out_valid
             assert hls_pool.attributes['pad_left'] == 0
             assert hls_pool.attributes['pad_right'] == 0
+
+    test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend)