From f1c952aa48125bce679515ed59d9847bc99b2ebf Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 15 Jan 2025 09:55:12 -0600 Subject: [PATCH 01/35] add conftest.py, utils.py, update test_keras_api.py --- test/pytest/conftest.py | 17 +++++++++++++++++ test/pytest/test_keras_api.py | 9 +++++++-- test/pytest/utils.py | 24 ++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 test/pytest/conftest.py create mode 100644 test/pytest/utils.py diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py new file mode 100644 index 0000000000..706efb004e --- /dev/null +++ b/test/pytest/conftest.py @@ -0,0 +1,17 @@ +import pytest +import os + + +def pytest_addoption(parser): + parser.addoption( + "--synthesis", action="store_true", default=False, help="Enable synthesis test" + ) + +@pytest.fixture +def synthesis(request): + """ + Fixture to determine if synthesis step should be run. + Reads the '--synthesis' command-line argument passed to pytest. + If the argument is provided, it will return True; otherwise, False. + """ + return request.config.getoption("--synthesis") diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index 4bb9f03751..d0b8f9d273 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -1,5 +1,6 @@ import math from pathlib import Path +from utils import check_synthesis import numpy as np import pytest @@ -27,7 +28,7 @@ @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_dense(backend, io_type): +def test_dense(backend, io_type, synthesis): model = tf.keras.models.Sequential() model.add( Dense( @@ -53,6 +54,7 @@ def test_dense(backend, io_type): config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') + baseline_path = str(test_root_path / f'keras_api_dense_{backend}_{io_type}') hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -67,13 +69,16 @@ def test_dense(backend, io_type): assert len(model.layers) + 1 == len(hls_model.get_layers()) assert list(hls_model.get_layers())[0].attributes['class_name'] == "InputLayer" assert list(hls_model.get_layers())[1].attributes["class_name"] == model.layers[0]._name - assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' + # assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' assert list(hls_model.get_layers())[0].attributes['input_shape'] == list(model.layers[0].input_shape[1:]) assert list(hls_model.get_layers())[1].attributes['n_in'] == model.layers[0].input_shape[1:][0] assert list(hls_model.get_layers())[1].attributes['n_out'] == model.layers[0].output_shape[1:][0] assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + + # TODO: add ThresholdedReLU test when it can be made to pass # https://github.com/fastmachinelearning/hls4ml/issues/376 diff --git a/test/pytest/utils.py b/test/pytest/utils.py new file mode 100644 index 0000000000..55a302fbe7 --- /dev/null +++ b/test/pytest/utils.py @@ -0,0 +1,24 @@ +import json +import os +import warnings +import json + + +def compare_synthesis(data, filename): + """ Compares given data to a baseline stored in the file. """ + with open(filename, "w") as fp: + baseline = json.dump(data, fp) + return data == baseline + +def check_synthesis(synthesis, hls_model, baseline_path): + """Function to run synthesis and compare results.""" + if synthesis: + data = hls_model.build() + if not compare_synthesis(data, baseline_path): + warnings.warn("Results don't match baseline") + if data.keys() < {'CSimResults', 'CSynthesisReport'}: + raise ValueError('Synthesis Failed') + + + + From 4fb8bfe9e64051864778c916e60f13504cde05d4 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 15 Jan 2025 09:56:39 -0600 Subject: [PATCH 02/35] Use asserts in check_synthesis --- test/pytest/utils.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/test/pytest/utils.py b/test/pytest/utils.py index 55a302fbe7..14670aa2a9 100644 --- a/test/pytest/utils.py +++ b/test/pytest/utils.py @@ -14,10 +14,14 @@ def check_synthesis(synthesis, hls_model, baseline_path): """Function to run synthesis and compare results.""" if synthesis: data = hls_model.build() - if not compare_synthesis(data, baseline_path): - warnings.warn("Results don't match baseline") - if data.keys() < {'CSimResults', 'CSynthesisReport'}: - raise ValueError('Synthesis Failed') + + # Assert that the synthesis results match the baseline + assert compare_synthesis(data, baseline_path), \ + "Synthesis results do not match the baseline" + + # Assert that the required keys are present in the synthesis data + assert {'CSimResults', 'CSynthesisReport'}.issubset(data.keys()), \ + "Synthesis failed: Missing expected keys in the synthesis report" From 38f9199a11a7a1a3d18e741158ec9fe190998d40 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 14 Mar 2025 13:28:20 -0500 Subject: [PATCH 03/35] update utils.py --- test/pytest/test_keras_api.py | 43 ++++++++++++++++++++++++++++------- test/pytest/utils.py | 27 +++++++++++++++++++++- 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index d0b8f9d273..317ec4ba7f 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -1,6 +1,6 @@ import math from pathlib import Path -from utils import check_synthesis +from utils import check_synthesis, get_baselines_dir import numpy as np import pytest @@ -24,6 +24,7 @@ import hls4ml test_root_path = Path(__file__).parent +baselines_path = get_baselines_dir() @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) @@ -54,7 +55,7 @@ def test_dense(backend, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') - baseline_path = str(test_root_path / f'keras_api_dense_{backend}_{io_type}') + baseline_path = str(baselines_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -97,7 +98,7 @@ def test_dense(backend, io_type, synthesis): # ThresholdedReLU(theta=1.0)]) @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_activations(activation_function, backend, io_type): +def test_activations(activation_function, backend, io_type, synthesis): model = tf.keras.models.Sequential() model.add(Dense(64, input_shape=(1,), name='Dense', kernel_initializer='lecun_uniform', kernel_regularizer=None)) model.add(activation_function) @@ -107,6 +108,8 @@ def test_activations(activation_function, backend, io_type): keras_prediction = model.predict(X_input) config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') + baseline_path = str(baselines_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -119,6 +122,8 @@ def test_activations(activation_function, backend, io_type): assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + padds_options = ['same', 'valid'] @@ -136,7 +141,7 @@ def test_activations(activation_function, backend, io_type): ], ) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_conv1d(padds, backend, strategy, io_type): +def test_conv1d(padds, backend, strategy, io_type, synthesis): model = tf.keras.models.Sequential() input_shape = (10, 128, 4) model.add( @@ -161,6 +166,8 @@ def test_conv1d(padds, backend, strategy, io_type): config = hls4ml.utils.config_from_keras_model(model) config['Model']['Strategy'] = strategy output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}') + baseline_path = str(baselines_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -200,6 +207,8 @@ def test_conv1d(padds, backend, strategy, io_type): elif model.layers[0].padding == 'valid': assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 + + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) chans_options = ['channels_last'] @@ -220,7 +229,7 @@ def test_conv1d(padds, backend, strategy, io_type): ], ) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_conv2d(chans, padds, backend, strategy, io_type): +def test_conv2d(chans, padds, backend, strategy, io_type, synthesis): model = tf.keras.models.Sequential() input_shape = (28, 28, 3) model.add( @@ -243,6 +252,8 @@ def test_conv2d(chans, padds, backend, strategy, io_type): config = hls4ml.utils.config_from_keras_model(model) config['Model']['Strategy'] = strategy output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}') + baseline_path = str(baselines_path / f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -320,12 +331,14 @@ def test_conv2d(chans, padds, backend, strategy, io_type): assert list(hls_model.get_layers())[1].attributes['pad_bottom'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 + + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) # Currently only Vivado and Vitis is supported for io_stream. @pytest.mark.parametrize('backend', ['Vivado', 'Vitis']) @pytest.mark.parametrize('io_type', ['io_stream']) -def test_depthwise2d(backend, io_type): +def test_depthwise2d(backend, io_type, synthesis): ''' Test proper handling of DepthwiseConv2D ''' @@ -339,6 +352,8 @@ def test_depthwise2d(backend, io_type): model, granularity='name', default_precision='fixed<32,12>', backend=backend ) output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}') + baseline_path = str(baselines_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -349,11 +364,13 @@ def test_depthwise2d(backend, io_type): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + # Currently only Vivado and Vitis is supported for io_stream. @pytest.mark.parametrize('backend', ['Vivado', 'Vitis']) @pytest.mark.parametrize('io_type', ['io_stream']) -def test_depthwise1d(backend, io_type): +def test_depthwise1d(backend, io_type, synthesis): ''' Test proper handling of DepthwiseConv1D. ''' @@ -365,6 +382,8 @@ def test_depthwise1d(backend, io_type): config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend) output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}') + baseline_path = str(baselines_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}') + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -375,6 +394,8 @@ def test_depthwise1d(backend, io_type): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + pooling_layers = [MaxPooling1D, MaxPooling2D, AveragePooling1D, AveragePooling2D] @@ -383,7 +404,7 @@ def test_depthwise1d(backend, io_type): @pytest.mark.parametrize('padds', padds_options) @pytest.mark.parametrize('chans', chans_options) @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) -def test_pooling(pooling, padds, chans, backend): +def test_pooling(pooling, padds, chans, backend, synthesis): assert '1D' in pooling.__name__ or '2D' in pooling.__name__ input_shape = (18, 15, 3) if '2D' in pooling.__name__ else (121, 3) @@ -397,6 +418,10 @@ def test_pooling(pooling, padds, chans, backend): output_dir = str( test_root_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' ) + baseline_path = str( + baselines_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' + ) + hls_model = hls4ml.converters.convert_from_keras_model( keras_model, hls_config=hls_cfg, output_dir=output_dir, backend=backend ) @@ -500,3 +525,5 @@ def test_pooling(pooling, padds, chans, backend): assert hls_pool.attributes['n_out'] == out_valid assert hls_pool.attributes['pad_left'] == 0 assert hls_pool.attributes['pad_right'] == 0 + + check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) diff --git a/test/pytest/utils.py b/test/pytest/utils.py index 14670aa2a9..7fa2eb7c10 100644 --- a/test/pytest/utils.py +++ b/test/pytest/utils.py @@ -1,15 +1,40 @@ import json import os -import warnings import json +def get_baselines_dir(): + """ + Returns the baselines directory path. + + Checks the `BASELINES_DIR` environment variable, which must contain an absolute + path to the baselines directory. If the environment variable is not set or is + invalid, raises a `ValueError`. + + Returns: + str: Absolute path to the baseline directory. + + Raises: + ValueError: If `BASELINES_DIR` is not set or is not a valid absolute path. + """ + baselines_dir = os.getenv('BASELINE_DIR') + + if not baselines_dir: + raise ValueError("The 'BASELINES_DIR' environment variable must be set.") + + if not os.path.isabs(baselines_dir): + raise ValueError(f"The path provided in 'BASELINES_DIR' must be an absolute path. Got: {baselines_dir}") + + return baselines_dir + + def compare_synthesis(data, filename): """ Compares given data to a baseline stored in the file. """ with open(filename, "w") as fp: baseline = json.dump(data, fp) return data == baseline + def check_synthesis(synthesis, hls_model, baseline_path): """Function to run synthesis and compare results.""" if synthesis: From 109195daa6df6f67035ec56d3b5a0da7fd0b6859 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 19 Mar 2025 10:45:40 -0500 Subject: [PATCH 04/35] fix baselines dir path --- test/pytest/test_keras_api.py | 2 +- test/pytest/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index 317ec4ba7f..f6cd98e233 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -24,7 +24,7 @@ import hls4ml test_root_path = Path(__file__).parent -baselines_path = get_baselines_dir() +baselines_path = Path(get_baselines_dir()) @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) diff --git a/test/pytest/utils.py b/test/pytest/utils.py index 7fa2eb7c10..01c221fb0a 100644 --- a/test/pytest/utils.py +++ b/test/pytest/utils.py @@ -17,7 +17,7 @@ def get_baselines_dir(): Raises: ValueError: If `BASELINES_DIR` is not set or is not a valid absolute path. """ - baselines_dir = os.getenv('BASELINE_DIR') + baselines_dir = os.getenv('BASELINES_DIR') if not baselines_dir: raise ValueError("The 'BASELINES_DIR' environment variable must be set.") From d25ea90b49e20f37812114e254540c247879e755 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 26 Mar 2025 11:08:45 -0500 Subject: [PATCH 05/35] refactor synthesis_helpers.py --- .gitmodules | 3 + test/pytest/baselines | 1 + test/pytest/ci-template.yml | 2 +- test/pytest/synthesis_helpers.py | 101 +++++++++++++++++++++++++++++++ test/pytest/test_keras_api.py | 34 +++++------ test/pytest/utils.py | 53 ---------------- 6 files changed, 121 insertions(+), 73 deletions(-) create mode 160000 test/pytest/baselines create mode 100644 test/pytest/synthesis_helpers.py delete mode 100644 test/pytest/utils.py diff --git a/.gitmodules b/.gitmodules index 98c3df68fd..9e228bb85d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "hls4ml/templates/catapult/ac_math"] path = hls4ml/templates/catapult/ac_math url = https://github.com/hlslibs/ac_math.git +[submodule "test/pytest/baselines"] + path = test/pytest/baselines + url = https://github.com/marco66colombo/baselines.git diff --git a/test/pytest/baselines b/test/pytest/baselines new file mode 160000 index 0000000000..5d00fbdda2 --- /dev/null +++ b/test/pytest/baselines @@ -0,0 +1 @@ +Subproject commit 5d00fbdda2d9209c8da53d6674b7a8f1ba7ae113 diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 873fe0fec4..cb86148985 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -13,7 +13,7 @@ - pip install .[testing,sr,optimization] script: - cd test/pytest - - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed --synthesis artifacts: when: always reports: diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py new file mode 100644 index 0000000000..f9eccb8ac8 --- /dev/null +++ b/test/pytest/synthesis_helpers.py @@ -0,0 +1,101 @@ +import json +import os +import pytest +from pathlib import Path + + +def save_baseline(data, filename): + """ Saves the given data as a baseline in the specified file. """ + with open(filename, "w") as fp: + json.dump(data, fp, indent=4) + + +def get_baseline_path(baseline_file_name, backend): + + tool_versions = { + 'Vivado': '2023.1', + 'Vitis': '2023.1', + } + + default_version = 'latest' + + version = tool_versions.get(backend, default_version) + + return ( + Path(__file__).parent / + "baselines" / backend / version / baseline_file_name + ) + + +def get_tolerance(key): + """ + Get the tolerance for a given key, using a predefined set of tolerances. + + :param key: The synthesis report key to check. + :return: The tolerance value for the given key. + """ + + tolerances = { + "EstimatedClockPeriod": 0.01, + "FF": 0.05, + "LUT": 0.10, + "BRAM_18K": 0.0, + "DSP": 0.0, + "URAM": 0.0, + "AvailableBRAM_18K": 0.0, + "AvailableDSP": 0.0, + "AvailableFF": 0.0, + "AvailableLUT": 0.0, + "AvailableURAM": 0.0, + } + + # Default tolerance for unspecified keys + default_tolerance = 0.01 + + return tolerances.get(key, default_tolerance) + + +def compare_synthesis_tolerance(data, filename): + """ Compare synthesis report values with a given tolerance. """ + try: + with open(filename, "r") as fp: + baseline = json.load(fp) + except FileNotFoundError: + pytest.skip(f"Baseline file '{filename}' not found.") + + csrBaseline = baseline.get("CSynthesisReport") + csrData = data.get("CSynthesisReport") + + for key, expected_value in csrBaseline.items(): + actual_value = csrData.get(key) + tolerance = get_tolerance(key) + + try: + # Convert to float for numerical comparison + expected_num = float(expected_value) + actual_num = float(actual_value) + assert actual_num == pytest.approx(expected_num, rel=tolerance), \ + f"{key}: expected {expected_num}, got {actual_num} (tolerance={tolerance*100}%)" + except ValueError: + # Exact match for non-numeric values + assert actual_value == expected_value, f"{key}: expected '{expected_value}', got '{actual_value}'" + + +def test_synthesis(synthesis, hls_model, baseline_file_name, backend): + """Function to run synthesis and compare results.""" + if synthesis: + + if hls_model.config.get_config_value('Backend') == 'oneAPI': + pytest.skip(f'oneAPI backend not supported in synthesis tests.') + + try: + # TODO: should csim be True? whaat other params should be set? + data = hls_model.build(csim=True) + except Exception as e: + pytest.skip(str(e)) + + assert {'CSimResults', 'CSynthesisReport'}.issubset(data.keys()), \ + "Synthesis failed: Missing expected keys in the synthesis report" + + baseline_path = get_baseline_path(baseline_file_name, backend) + compare_synthesis_tolerance(data, baseline_path) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index f6cd98e233..f8f26764eb 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -1,7 +1,6 @@ import math from pathlib import Path -from utils import check_synthesis, get_baselines_dir - +from synthesis_helpers import test_synthesis import numpy as np import pytest import tensorflow as tf @@ -24,7 +23,6 @@ import hls4ml test_root_path = Path(__file__).parent -baselines_path = Path(get_baselines_dir()) @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) @@ -55,7 +53,7 @@ def test_dense(backend, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') - baseline_path = str(baselines_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -77,7 +75,7 @@ def test_dense(backend, io_type, synthesis): assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) @@ -108,7 +106,7 @@ def test_activations(activation_function, backend, io_type, synthesis): keras_prediction = model.predict(X_input) config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') - baseline_path = str(baselines_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -122,7 +120,7 @@ def test_activations(activation_function, backend, io_type, synthesis): assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) padds_options = ['same', 'valid'] @@ -166,7 +164,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) config['Model']['Strategy'] = strategy output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}') - baseline_path = str(baselines_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -208,7 +206,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis): assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) chans_options = ['channels_last'] @@ -252,7 +250,7 @@ def test_conv2d(chans, padds, backend, strategy, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) config['Model']['Strategy'] = strategy output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}') - baseline_path = str(baselines_path / f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -332,7 +330,7 @@ def test_conv2d(chans, padds, backend, strategy, io_type, synthesis): assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # Currently only Vivado and Vitis is supported for io_stream. @@ -352,7 +350,7 @@ def test_depthwise2d(backend, io_type, synthesis): model, granularity='name', default_precision='fixed<32,12>', backend=backend ) output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}') - baseline_path = str(baselines_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -364,7 +362,7 @@ def test_depthwise2d(backend, io_type, synthesis): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # Currently only Vivado and Vitis is supported for io_stream. @@ -382,7 +380,7 @@ def test_depthwise1d(backend, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend) output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}') - baseline_path = str(baselines_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -394,7 +392,7 @@ def test_depthwise1d(backend, io_type, synthesis): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) pooling_layers = [MaxPooling1D, MaxPooling2D, AveragePooling1D, AveragePooling2D] @@ -418,9 +416,7 @@ def test_pooling(pooling, padds, chans, backend, synthesis): output_dir = str( test_root_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' ) - baseline_path = str( - baselines_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' - ) + baseline_file_name = f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' hls_model = hls4ml.converters.convert_from_keras_model( keras_model, hls_config=hls_cfg, output_dir=output_dir, backend=backend @@ -526,4 +522,4 @@ def test_pooling(pooling, padds, chans, backend, synthesis): assert hls_pool.attributes['pad_left'] == 0 assert hls_pool.attributes['pad_right'] == 0 - check_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_path=baseline_path) + test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) diff --git a/test/pytest/utils.py b/test/pytest/utils.py deleted file mode 100644 index 01c221fb0a..0000000000 --- a/test/pytest/utils.py +++ /dev/null @@ -1,53 +0,0 @@ -import json -import os -import json - - -def get_baselines_dir(): - """ - Returns the baselines directory path. - - Checks the `BASELINES_DIR` environment variable, which must contain an absolute - path to the baselines directory. If the environment variable is not set or is - invalid, raises a `ValueError`. - - Returns: - str: Absolute path to the baseline directory. - - Raises: - ValueError: If `BASELINES_DIR` is not set or is not a valid absolute path. - """ - baselines_dir = os.getenv('BASELINES_DIR') - - if not baselines_dir: - raise ValueError("The 'BASELINES_DIR' environment variable must be set.") - - if not os.path.isabs(baselines_dir): - raise ValueError(f"The path provided in 'BASELINES_DIR' must be an absolute path. Got: {baselines_dir}") - - return baselines_dir - - -def compare_synthesis(data, filename): - """ Compares given data to a baseline stored in the file. """ - with open(filename, "w") as fp: - baseline = json.dump(data, fp) - return data == baseline - - -def check_synthesis(synthesis, hls_model, baseline_path): - """Function to run synthesis and compare results.""" - if synthesis: - data = hls_model.build() - - # Assert that the synthesis results match the baseline - assert compare_synthesis(data, baseline_path), \ - "Synthesis results do not match the baseline" - - # Assert that the required keys are present in the synthesis data - assert {'CSimResults', 'CSynthesisReport'}.issubset(data.keys()), \ - "Synthesis failed: Missing expected keys in the synthesis report" - - - - From f2abbeb88b8883e8370bb5953fc8ade0c5c286d0 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 26 Mar 2025 11:11:46 -0500 Subject: [PATCH 06/35] fix comment --- test/pytest/test_keras_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index f8f26764eb..687c178662 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -68,7 +68,7 @@ def test_dense(backend, io_type, synthesis): assert len(model.layers) + 1 == len(hls_model.get_layers()) assert list(hls_model.get_layers())[0].attributes['class_name'] == "InputLayer" assert list(hls_model.get_layers())[1].attributes["class_name"] == model.layers[0]._name - # assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' + assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' assert list(hls_model.get_layers())[0].attributes['input_shape'] == list(model.layers[0].input_shape[1:]) assert list(hls_model.get_layers())[1].attributes['n_in'] == model.layers[0].input_shape[1:][0] assert list(hls_model.get_layers())[1].attributes['n_out'] == model.layers[0].output_shape[1:][0] From a7bbc04aee6b5e1c181df1b41462de25ec72b1fc Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 13:44:43 -0500 Subject: [PATCH 07/35] pytest synthesis tests refactor --- test/pytest/ci-template.yml | 3 +- test/pytest/conftest.py | 36 +++++++--- test/pytest/synthesis_helpers.py | 118 ++++++++++++++++++++----------- test/pytest/test_keras_api.py | 43 +++++------ 4 files changed, 127 insertions(+), 73 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index cb86148985..5db61850c9 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -13,7 +13,7 @@ - pip install .[testing,sr,optimization] script: - cd test/pytest - - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed --synthesis + - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed artifacts: when: always reports: @@ -24,3 +24,4 @@ path: test/pytest/coverage.xml paths: - test/pytest/hls4mlprj*.tar.gz + - test/pytest/synthesis_report_*.json diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 706efb004e..ed5af778a1 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -2,16 +2,32 @@ import os -def pytest_addoption(parser): - parser.addoption( - "--synthesis", action="store_true", default=False, help="Enable synthesis test" - ) +def str_to_bool(val): + return str(val).lower() in ("1", "true") -@pytest.fixture -def synthesis(request): +@pytest.fixture(scope="module") +def synthesis_config(): """ - Fixture to determine if synthesis step should be run. - Reads the '--synthesis' command-line argument passed to pytest. - If the argument is provided, it will return True; otherwise, False. + Fixture that provides synthesis configuration for tests. + + It gathers: + - Whether synthesis should be run (from the RUN_SYNTHESIS env var) + - Tool versions for each supported backend (from env vars) + - Build arguments specific to each backend toolchain + """ - return request.config.getoption("--synthesis") + return { + "run_synthesis": str_to_bool(os.getenv("RUN_SYNTHESIS", "false")), + "tools_version": { + "Vivado": os.getenv("VIVADO_VERSION", "2020.1"), + "Vitis": os.getenv("VITIS_VERSION", "2021.2"), + "Quartus": os.getenv("QUARTUS_VERSION", "default"), + "oneAPI": os.getenv("ONEAPI_VERSION", "default"), + }, + "build_args": { + "Vivado": {"csim": False, "synth": True, "export": False}, + "Vitis": {"csim": False, "synth": True, "export": False}, + "Quartus": {"synth": True, "fpgasynth": False}, + "oneAPI": {"build_type": "fpga_emu", "run": False} + } + } \ No newline at end of file diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index f9eccb8ac8..dbf9d57998 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -4,37 +4,50 @@ from pathlib import Path -def save_baseline(data, filename): - """ Saves the given data as a baseline in the specified file. """ - with open(filename, "w") as fp: - json.dump(data, fp, indent=4) - - -def get_baseline_path(baseline_file_name, backend): - - tool_versions = { - 'Vivado': '2023.1', - 'Vitis': '2023.1', - } - - default_version = 'latest' +def get_baseline_path(baseline_file_name, backend, version): + """ + Construct the full path to a baseline synthesis report file. - version = tool_versions.get(backend, default_version) + Args: + baseline_file_name (str): The name of the baseline report file. + backend (str): The backend used (e.g., 'Vivado', 'Vitis'). + version (str): The tool version (e.g., '2020.1'). + Returns: + Path: A pathlib.Path object pointing to the baseline file location. + """ return ( Path(__file__).parent / "baselines" / backend / version / baseline_file_name ) -def get_tolerance(key): +def save_report(data, filename): """ - Get the tolerance for a given key, using a predefined set of tolerances. + Save synthesis data to a JSON file in the same directory as this script. - :param key: The synthesis report key to check. - :return: The tolerance value for the given key. + Args: + data (dict): The synthesis output data to be saved. + filename (str): The filename to write to (e.g., 'synthesis_report_test_x.json'). + + Raises: + OSError: If the file cannot be written. + """ + out_path = Path(__file__).parent / filename + with open(out_path, "w") as fp: + json.dump(data, fp, indent=4) + + +def get_tolerance(key): """ + Get the relative tolerance for a specific synthesis report field. + Args: + key (str): The key in the synthesis report to compare. + + Returns: + float: The relative tolerance allowed for that key. Defaults to 1% (0.01). + """ tolerances = { "EstimatedClockPeriod": 0.01, "FF": 0.05, @@ -49,20 +62,19 @@ def get_tolerance(key): "AvailableURAM": 0.0, } - # Default tolerance for unspecified keys default_tolerance = 0.01 return tolerances.get(key, default_tolerance) -def compare_synthesis_tolerance(data, filename): - """ Compare synthesis report values with a given tolerance. """ - try: - with open(filename, "r") as fp: - baseline = json.load(fp) - except FileNotFoundError: - pytest.skip(f"Baseline file '{filename}' not found.") +def compare_reports_with_tolerance(data, baseline): + """ + Compare two synthesis reports using tolerances defined per key. + Args: + data (dict): The current synthesis report. + baseline (dict): The baseline synthesis report to compare against. + """ csrBaseline = baseline.get("CSynthesisReport") csrData = data.get("CSynthesisReport") @@ -81,21 +93,45 @@ def compare_synthesis_tolerance(data, filename): assert actual_value == expected_value, f"{key}: expected '{expected_value}', got '{actual_value}'" -def test_synthesis(synthesis, hls_model, baseline_file_name, backend): - """Function to run synthesis and compare results.""" - if synthesis: +def test_synthesis(config, hls_model, baseline_file_name, backend): + """ + Run HLS synthesis and compare the output with a stored baseline report. - if hls_model.config.get_config_value('Backend') == 'oneAPI': - pytest.skip(f'oneAPI backend not supported in synthesis tests.') + If synthesis is disabled via the configuration (`run_synthesis=False`), + no synthesis is executed and the test silently returns. - try: - # TODO: should csim be True? whaat other params should be set? - data = hls_model.build(csim=True) - except Exception as e: - pytest.skip(str(e)) + Args: + config (dict): Test-wide synthesis configuration fixture. + hls_model (object): hls4ml model instance to build and synthesize. + baseline_file_name (str): The name of the baseline file for comparison. + backend (str): The synthesis backend used (e.g., 'Vivado', 'Vitis'). + """ + if not config.get("run_synthesis", False): + # TODO: should this info be printed or logged? + return + + if backend == 'oneAPI': + pytest.skip(f'oneAPI backend not supported in synthesis tests.') - assert {'CSimResults', 'CSynthesisReport'}.issubset(data.keys()), \ - "Synthesis failed: Missing expected keys in the synthesis report" + build_args = config["build_args"] + + try: + data = hls_model.build(**build_args.get(backend, {})) + except Exception as e: + pytest.skip(f"hls_model.build failed: {e}") + + assert {'CSimResults', 'CSynthesisReport'}.issubset(data.keys()), \ + "Synthesis failed: Missing expected keys in the synthesis report" + + save_report(data, f"synthesis_report_{baseline_file_name}") + + version = config["tools_version"].get(backend) + baseline_path = get_baseline_path(baseline_file_name, backend, version) + + try: + with open(baseline_path, "r") as fp: + baseline = json.load(fp) + except FileNotFoundError: + pytest.skip(f"Baseline file '{baseline_path}' not found.") - baseline_path = get_baseline_path(baseline_file_name, backend) - compare_synthesis_tolerance(data, baseline_path) + compare_reports_with_tolerance(data, baseline) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index 687c178662..e25b052bdd 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -19,6 +19,7 @@ MaxPooling2D, PReLU, ) +import os import hls4ml @@ -27,7 +28,7 @@ @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_dense(backend, io_type, synthesis): +def test_dense(backend, io_type, synthesis_config): model = tf.keras.models.Sequential() model.add( Dense( @@ -53,8 +54,8 @@ def test_dense(backend, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}' - + baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}.json' + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -75,7 +76,7 @@ def test_dense(backend, io_type, synthesis): assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) @@ -96,7 +97,7 @@ def test_dense(backend, io_type, synthesis): # ThresholdedReLU(theta=1.0)]) @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_activations(activation_function, backend, io_type, synthesis): +def test_activations(activation_function, backend, io_type, synthesis_config): model = tf.keras.models.Sequential() model.add(Dense(64, input_shape=(1,), name='Dense', kernel_initializer='lecun_uniform', kernel_regularizer=None)) model.add(activation_function) @@ -120,7 +121,7 @@ def test_activations(activation_function, backend, io_type, synthesis): assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) padds_options = ['same', 'valid'] @@ -139,7 +140,7 @@ def test_activations(activation_function, backend, io_type, synthesis): ], ) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_conv1d(padds, backend, strategy, io_type, synthesis): +def test_conv1d(padds, backend, strategy, io_type, synthesis_config): model = tf.keras.models.Sequential() input_shape = (10, 128, 4) model.add( @@ -164,7 +165,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) config['Model']['Strategy'] = strategy output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}' + baseline_file_name = f'hls4mlprj_keras_api_conv1d_{padds}_{backend}_{strategy}_{io_type}.json' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -206,7 +207,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis): assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) chans_options = ['channels_last'] @@ -227,7 +228,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis): ], ) @pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_conv2d(chans, padds, backend, strategy, io_type, synthesis): +def test_conv2d(chans, padds, backend, strategy, io_type, synthesis_config): model = tf.keras.models.Sequential() input_shape = (28, 28, 3) model.add( @@ -250,7 +251,7 @@ def test_conv2d(chans, padds, backend, strategy, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model) config['Model']['Strategy'] = strategy output_dir = str(test_root_path / f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}' + baseline_file_name = f'hls4mlprj_keras_api_conv2d_{backend}_{strategy}_{chans}_{padds}_{io_type}.json' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -330,13 +331,13 @@ def test_conv2d(chans, padds, backend, strategy, io_type, synthesis): assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # Currently only Vivado and Vitis is supported for io_stream. @pytest.mark.parametrize('backend', ['Vivado', 'Vitis']) @pytest.mark.parametrize('io_type', ['io_stream']) -def test_depthwise2d(backend, io_type, synthesis): +def test_depthwise2d(backend, io_type, synthesis_config): ''' Test proper handling of DepthwiseConv2D ''' @@ -350,7 +351,7 @@ def test_depthwise2d(backend, io_type, synthesis): model, granularity='name', default_precision='fixed<32,12>', backend=backend ) output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}' + baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv2d_{backend}_{io_type}.json' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -362,13 +363,13 @@ def test_depthwise2d(backend, io_type, synthesis): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # Currently only Vivado and Vitis is supported for io_stream. @pytest.mark.parametrize('backend', ['Vivado', 'Vitis']) @pytest.mark.parametrize('io_type', ['io_stream']) -def test_depthwise1d(backend, io_type, synthesis): +def test_depthwise1d(backend, io_type, synthesis_config): ''' Test proper handling of DepthwiseConv1D. ''' @@ -380,7 +381,7 @@ def test_depthwise1d(backend, io_type, synthesis): config = hls4ml.utils.config_from_keras_model(model, granularity='name', backend=backend) output_dir = str(test_root_path / f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}' + baseline_file_name = f'hls4mlprj_keras_api_depthwiseconv1d_{backend}_{io_type}.json' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type @@ -392,7 +393,7 @@ def test_depthwise1d(backend, io_type, synthesis): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) pooling_layers = [MaxPooling1D, MaxPooling2D, AveragePooling1D, AveragePooling2D] @@ -402,7 +403,7 @@ def test_depthwise1d(backend, io_type, synthesis): @pytest.mark.parametrize('padds', padds_options) @pytest.mark.parametrize('chans', chans_options) @pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) -def test_pooling(pooling, padds, chans, backend, synthesis): +def test_pooling(pooling, padds, chans, backend, synthesis_config): assert '1D' in pooling.__name__ or '2D' in pooling.__name__ input_shape = (18, 15, 3) if '2D' in pooling.__name__ else (121, 3) @@ -416,7 +417,7 @@ def test_pooling(pooling, padds, chans, backend, synthesis): output_dir = str( test_root_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' ) - baseline_file_name = f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' + baseline_file_name = f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}.json' hls_model = hls4ml.converters.convert_from_keras_model( keras_model, hls_config=hls_cfg, output_dir=output_dir, backend=backend @@ -522,4 +523,4 @@ def test_pooling(pooling, padds, chans, backend, synthesis): assert hls_pool.attributes['pad_left'] == 0 assert hls_pool.attributes['pad_right'] == 0 - test_synthesis(synthesis=synthesis, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From 343a4e19d36474b97f46006e3178466cec6da7ca Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 15:04:35 -0500 Subject: [PATCH 08/35] update synthesis_helpers, confetst.py --- test/pytest/conftest.py | 4 ++-- test/pytest/synthesis_helpers.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index ed5af778a1..43cda62756 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -21,8 +21,8 @@ def synthesis_config(): "tools_version": { "Vivado": os.getenv("VIVADO_VERSION", "2020.1"), "Vitis": os.getenv("VITIS_VERSION", "2021.2"), - "Quartus": os.getenv("QUARTUS_VERSION", "default"), - "oneAPI": os.getenv("ONEAPI_VERSION", "default"), + "Quartus": os.getenv("QUARTUS_VERSION", "latest"), + "oneAPI": os.getenv("ONEAPI_VERSION", "latest"), }, "build_args": { "Vivado": {"csim": False, "synth": True, "export": False}, diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index dbf9d57998..cf4d83fa01 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -120,11 +120,11 @@ def test_synthesis(config, hls_model, baseline_file_name, backend): except Exception as e: pytest.skip(f"hls_model.build failed: {e}") - assert {'CSimResults', 'CSynthesisReport'}.issubset(data.keys()), \ - "Synthesis failed: Missing expected keys in the synthesis report" - save_report(data, f"synthesis_report_{baseline_file_name}") + assert data and {'CSynthesisReport'}.issubset(data.keys()), \ + "Synthesis failed: Missing expected keys in the synthesis report" + version = config["tools_version"].get(backend) baseline_path = get_baseline_path(baseline_file_name, backend, version) From 08abec395f99cba8bbd0c6881c982077fbf0a14f Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 15:07:34 -0500 Subject: [PATCH 09/35] update default vitis version --- test/pytest/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 43cda62756..ea1f459744 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -20,7 +20,7 @@ def synthesis_config(): "run_synthesis": str_to_bool(os.getenv("RUN_SYNTHESIS", "false")), "tools_version": { "Vivado": os.getenv("VIVADO_VERSION", "2020.1"), - "Vitis": os.getenv("VITIS_VERSION", "2021.2"), + "Vitis": os.getenv("VITIS_VERSION", "2020.1"), "Quartus": os.getenv("QUARTUS_VERSION", "latest"), "oneAPI": os.getenv("ONEAPI_VERSION", "latest"), }, From 3dae27bcfc91228857b95f72fe4e887e46dd7715 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 15:09:56 -0500 Subject: [PATCH 10/35] fix typo test_keras_api.py --- test/pytest/test_keras_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index e25b052bdd..4e0a316d18 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -107,7 +107,7 @@ def test_activations(activation_function, backend, io_type, synthesis_config): keras_prediction = model.predict(X_input) config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}' + baseline_file_name = f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}.json' hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type From 3195bcb0534f321729f2e7447c341e08c12453c8 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 16:02:35 -0500 Subject: [PATCH 11/35] clean imports test_keras_api --- test/pytest/test_keras_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index 4e0a316d18..0615ff1abf 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -1,5 +1,6 @@ import math from pathlib import Path + from synthesis_helpers import test_synthesis import numpy as np import pytest @@ -19,7 +20,6 @@ MaxPooling2D, PReLU, ) -import os import hls4ml From 5e8990befb1fb839fed3e7bf6f1b109cbaf649e6 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 16:06:13 -0500 Subject: [PATCH 12/35] clean new lines test_keras_api --- test/pytest/test_keras_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index 0615ff1abf..af9ee43e53 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -79,7 +79,6 @@ def test_dense(backend, io_type, synthesis_config): test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) - # TODO: add ThresholdedReLU test when it can be made to pass # https://github.com/fastmachinelearning/hls4ml/issues/376 @pytest.mark.parametrize( From c1436a9ffb64effd22fc27ed8fbe97e2e08dbdbf Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 16:06:53 -0500 Subject: [PATCH 13/35] Trigger pre-commit hook From 943dc74d83ccf529ba746722e3a94fb398acff9c Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 1 Apr 2025 16:29:27 -0500 Subject: [PATCH 14/35] update after precommit --- test/pytest/conftest.py | 10 ++++++---- test/pytest/synthesis_helpers.py | 29 ++++++++++++++--------------- test/pytest/test_keras_api.py | 16 +++++++++------- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index ea1f459744..77a924ed10 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -1,10 +1,12 @@ -import pytest import os +import pytest + def str_to_bool(val): return str(val).lower() in ("1", "true") + @pytest.fixture(scope="module") def synthesis_config(): """ @@ -28,6 +30,6 @@ def synthesis_config(): "Vivado": {"csim": False, "synth": True, "export": False}, "Vitis": {"csim": False, "synth": True, "export": False}, "Quartus": {"synth": True, "fpgasynth": False}, - "oneAPI": {"build_type": "fpga_emu", "run": False} - } - } \ No newline at end of file + "oneAPI": {"build_type": "fpga_emu", "run": False}, + }, + } diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index cf4d83fa01..85492d082c 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -1,8 +1,8 @@ import json -import os -import pytest from pathlib import Path +import pytest + def get_baseline_path(baseline_file_name, backend, version): """ @@ -16,10 +16,7 @@ def get_baseline_path(baseline_file_name, backend, version): Returns: Path: A pathlib.Path object pointing to the baseline file location. """ - return ( - Path(__file__).parent / - "baselines" / backend / version / baseline_file_name - ) + return Path(__file__).parent / "baselines" / backend / version / baseline_file_name def save_report(data, filename): @@ -62,7 +59,7 @@ def get_tolerance(key): "AvailableURAM": 0.0, } - default_tolerance = 0.01 + default_tolerance = 0.01 return tolerances.get(key, default_tolerance) @@ -86,8 +83,9 @@ def compare_reports_with_tolerance(data, baseline): # Convert to float for numerical comparison expected_num = float(expected_value) actual_num = float(actual_value) - assert actual_num == pytest.approx(expected_num, rel=tolerance), \ - f"{key}: expected {expected_num}, got {actual_num} (tolerance={tolerance*100}%)" + assert actual_num == pytest.approx( + expected_num, rel=tolerance + ), f"{key}: expected {expected_num}, got {actual_num} (tolerance={tolerance*100}%)" except ValueError: # Exact match for non-numeric values assert actual_value == expected_value, f"{key}: expected '{expected_value}', got '{actual_value}'" @@ -97,7 +95,7 @@ def test_synthesis(config, hls_model, baseline_file_name, backend): """ Run HLS synthesis and compare the output with a stored baseline report. - If synthesis is disabled via the configuration (`run_synthesis=False`), + If synthesis is disabled via the configuration (`run_synthesis=False`), no synthesis is executed and the test silently returns. Args: @@ -109,9 +107,9 @@ def test_synthesis(config, hls_model, baseline_file_name, backend): if not config.get("run_synthesis", False): # TODO: should this info be printed or logged? return - + if backend == 'oneAPI': - pytest.skip(f'oneAPI backend not supported in synthesis tests.') + pytest.skip('oneAPI backend not supported in synthesis tests.') build_args = config["build_args"] @@ -122,14 +120,15 @@ def test_synthesis(config, hls_model, baseline_file_name, backend): save_report(data, f"synthesis_report_{baseline_file_name}") - assert data and {'CSynthesisReport'}.issubset(data.keys()), \ - "Synthesis failed: Missing expected keys in the synthesis report" + assert data and {'CSynthesisReport'}.issubset( + data.keys() + ), "Synthesis failed: Missing expected keys in the synthesis report" version = config["tools_version"].get(backend) baseline_path = get_baseline_path(baseline_file_name, backend, version) try: - with open(baseline_path, "r") as fp: + with open(baseline_path) as fp: baseline = json.load(fp) except FileNotFoundError: pytest.skip(f"Baseline file '{baseline_path}' not found.") diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index af9ee43e53..772a7bcaed 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -1,10 +1,10 @@ import math from pathlib import Path -from synthesis_helpers import test_synthesis import numpy as np import pytest import tensorflow as tf +from synthesis_helpers import test_synthesis from tensorflow.keras.layers import ( ELU, Activation, @@ -55,7 +55,7 @@ def test_dense(backend, io_type, synthesis_config): config = hls4ml.utils.config_from_keras_model(model) output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}.json' - + hls_model = hls4ml.converters.convert_from_keras_model( model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type ) @@ -205,7 +205,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis_config): elif model.layers[0].padding == 'valid': assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) @@ -329,7 +329,7 @@ def test_conv2d(chans, padds, backend, strategy, io_type, synthesis_config): assert list(hls_model.get_layers())[1].attributes['pad_bottom'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) @@ -416,8 +416,10 @@ def test_pooling(pooling, padds, chans, backend, synthesis_config): output_dir = str( test_root_path / f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}' ) - baseline_file_name = f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}.json' - + baseline_file_name = ( + f'hls4mlprj_keras_api_pooling_{pooling.__name__}_channels_{chans}_padds_{padds}_backend_{backend}.json' + ) + hls_model = hls4ml.converters.convert_from_keras_model( keras_model, hls_config=hls_cfg, output_dir=output_dir, backend=backend ) @@ -521,5 +523,5 @@ def test_pooling(pooling, padds, chans, backend, synthesis_config): assert hls_pool.attributes['n_out'] == out_valid assert hls_pool.attributes['pad_left'] == 0 assert hls_pool.attributes['pad_right'] == 0 - + test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From 34b28eae170ed3d64c916bd034c94a5feb237a3a Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 3 Apr 2025 09:02:59 -0500 Subject: [PATCH 15/35] enable vivado and vitis --- test/pytest/ci-template.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 5db61850c9..776936ea62 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -11,6 +11,10 @@ - git submodule update --init --recursive hls4ml/templates/catapult/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] + - sudo yum install libtinfo.so.6 -y + - sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 + - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx + - source /opt/Xilinx/Vivado/2020.1/settings64.sh script: - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed From 290805280cba14362b3badd1f6d0d0ed9b77a469 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 9 Apr 2025 16:57:30 -0500 Subject: [PATCH 16/35] add oneAPI report support, bug fix --- test/pytest/ci-template.yml | 2 + test/pytest/conftest.py | 4 +- test/pytest/generate_ci_yaml.py | 72 ++----------------- test/pytest/synthesis_helpers.py | 115 +++++++++++++++++++++---------- test/pytest/test_keras_api.py | 16 ++--- 5 files changed, 98 insertions(+), 111 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 776936ea62..08e398ff29 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -9,12 +9,14 @@ - source /opt/intel/oneapi/setvars.sh --force - git config --global --add safe.directory /builds/fastmachinelearning/hls4ml - git submodule update --init --recursive hls4ml/templates/catapult/ + - git submodule update --init --recursive test/pytest/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] - sudo yum install libtinfo.so.6 -y - sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx - source /opt/Xilinx/Vivado/2020.1/settings64.sh + - export RUN_SYNTHESIS=true script: - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 77a924ed10..752816bbb1 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -24,12 +24,12 @@ def synthesis_config(): "Vivado": os.getenv("VIVADO_VERSION", "2020.1"), "Vitis": os.getenv("VITIS_VERSION", "2020.1"), "Quartus": os.getenv("QUARTUS_VERSION", "latest"), - "oneAPI": os.getenv("ONEAPI_VERSION", "latest"), + "oneAPI": os.getenv("ONEAPI_VERSION", "2025.0.1"), }, "build_args": { "Vivado": {"csim": False, "synth": True, "export": False}, "Vitis": {"csim": False, "synth": True, "export": False}, "Quartus": {"synth": True, "fpgasynth": False}, - "oneAPI": {"build_type": "fpga_emu", "run": False}, + "oneAPI": {"build_type": "report", "run": False}, }, } diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index adc3d680ab..74c8ef6957 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -1,7 +1,3 @@ -import itertools -import os -from pathlib import Path - import yaml ''' @@ -11,74 +7,20 @@ template = """ -pytest.{}: +pytest.test_cvmfs_mount: extends: .pytest variables: - PYTESTFILE: {} - EXAMPLEMODEL: {} + PYTESTFILE: test_keras_api.py + EXAMPLEMODEL: 0 """ -n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4)) - -# Blacklisted tests will be skipped -BLACKLIST = {'test_reduction'} - -# Long-running tests will not be bundled with other tests -LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} - - -def path_to_name(test_path): - path = Path(test_path) - name = path.stem.replace('test_', '') - return name - - -def batched(iterable, chunk_size): - iterator = iter(iterable) - while chunk := tuple(itertools.islice(iterator, chunk_size)): - yield chunk - - -def uses_example_model(test_filename): - with open(test_filename) as f: - content = f.read() - return 'example-models' in content - - -def generate_test_yaml(test_root='.'): - test_root = Path(test_root) - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem not in (BLACKLIST | LONGLIST)] - need_example_models = [uses_example_model(path) for path in test_paths] - - idxs = list(range(len(need_example_models))) - idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}') - - yml = None - for batch_idxs in batched(idxs, n_test_files_per_yml): - batch_paths: list[Path] = [test_paths[i] for i in batch_idxs] - names = [path_to_name(path) for path in batch_paths] - name = '+'.join(names) - test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths]) - batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs])) - diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model)) - if yml is None: - yml = diff_yml - else: - yml.update(diff_yml) - - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST] - for path in test_paths: - name = path.stem.replace('test_', '') - test_file = str(path.relative_to(test_root)) - needs_examples = uses_example_model(path) - diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples))) - yml.update(diff_yml) - +def generate_fixed_test_yaml(): + yml = yaml.safe_load(template) return yml if __name__ == '__main__': - yml = generate_test_yaml(Path(__file__).parent) + yml = generate_fixed_test_yaml() with open('pytests.yml', 'w') as yamlfile: - yaml.safe_dump(yml, yamlfile) + yaml.safe_dump(yml, yamlfile, default_flow_style=False) diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index 85492d082c..02de79ab97 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -35,16 +35,41 @@ def save_report(data, filename): json.dump(data, fp, indent=4) -def get_tolerance(key): +def compare_dicts(data, baseline, tolerances): """ - Get the relative tolerance for a specific synthesis report field. + Compare two flat dictionaries with tolerances. Args: - key (str): The key in the synthesis report to compare. + report (dict): The generated report dictionary. + baseline (dict): The expected/baseline dictionary. + tolerances (dict): Dictionary of tolerances per key. - Returns: - float: The relative tolerance allowed for that key. Defaults to 1% (0.01). + Raises: + AssertionError: If values differ outside the allowed tolerance. + """ + for key, expected in baseline.items(): + actual = data.get(key) + tolerance = tolerances.get(key, 0) + + try: + actual = float(actual) + expected = float(expected) + assert actual == pytest.approx( + expected, rel=tolerance + ), f"{key}: expected {expected}, got {actual} (tolerance={tolerance*100}%)" + except ValueError: + assert actual == expected, f"{key}: expected '{expected}', got '{actual}'" + + +def compare_vitis_backend(data, baseline): """ + Compare reports from Vivado/Vitis backends. + + Args: + data (dict): The current synthesis report. + baseline (dict): The expected synthesis report. + """ + tolerances = { "EstimatedClockPeriod": 0.01, "FF": 0.05, @@ -59,44 +84,54 @@ def get_tolerance(key): "AvailableURAM": 0.0, } - default_tolerance = 0.01 - - return tolerances.get(key, default_tolerance) + compare_dicts(data["CSynthesisReport"], baseline["CSynthesisReport"], tolerances) -def compare_reports_with_tolerance(data, baseline): +def compare_oneapi_backend(data, baseline): """ - Compare two synthesis reports using tolerances defined per key. + Compare reports from the oneAPI backend. Args: data (dict): The current synthesis report. - baseline (dict): The baseline synthesis report to compare against. + baseline (dict): The expected synthesis report. """ - csrBaseline = baseline.get("CSynthesisReport") - csrData = data.get("CSynthesisReport") - for key, expected_value in csrBaseline.items(): - actual_value = csrData.get(key) - tolerance = get_tolerance(key) + tolerances = { + "HLS": { + "total": {"alut": 0.01, "reg": 0.1, "ram": 0.01, "dsp": 0.01, "mlab": 0.01}, + "available": {"alut": 0.01, "reg": 0.01, "ram": 0.01, "dsp": 0.01, "mlab": 0.01}, + }, + "Loop": {"worstFrequency": 0.01, "worstII": 0.01, "worstLatency": 0.01}, + } - try: - # Convert to float for numerical comparison - expected_num = float(expected_value) - actual_num = float(actual_value) - assert actual_num == pytest.approx( - expected_num, rel=tolerance - ), f"{key}: expected {expected_num}, got {actual_num} (tolerance={tolerance*100}%)" - except ValueError: - # Exact match for non-numeric values - assert actual_value == expected_value, f"{key}: expected '{expected_value}', got '{actual_value}'" + data = data["report"] + baseline = baseline["report"] + + compare_dicts(data["HLS"]["total"], baseline["HLS"]["total"], tolerances["HLS"]["total"]) + compare_dicts(data["HLS"]["available"], baseline["HLS"]["available"], tolerances["HLS"]["available"]) + compare_dicts(data["Loop"], baseline["Loop"], tolerances["Loop"]) + + +COMPARE_FUNCS = { + "Vivado": compare_vitis_backend, + "Vitis": compare_vitis_backend, + "oneAPI": compare_oneapi_backend, +} -def test_synthesis(config, hls_model, baseline_file_name, backend): +EXPECTED_REPORT_KEYS = { + "Vivado": {"CSynthesisReport"}, + "Vitis": {"CSynthesisReport"}, + "oneAPI": {"report"}, +} + + +def run_synthesis_test(config, hls_model, baseline_file_name, backend): """ Run HLS synthesis and compare the output with a stored baseline report. If synthesis is disabled via the configuration (`run_synthesis=False`), - no synthesis is executed and the test silently returns. + no synthesis is executed and the method silently returns. Args: config (dict): Test-wide synthesis configuration fixture. @@ -105,32 +140,40 @@ def test_synthesis(config, hls_model, baseline_file_name, backend): backend (str): The synthesis backend used (e.g., 'Vivado', 'Vitis'). """ if not config.get("run_synthesis", False): - # TODO: should this info be printed or logged? return - if backend == 'oneAPI': - pytest.skip('oneAPI backend not supported in synthesis tests.') + # Skip Quartus backend + if backend == 'Quartus': + return + # Run synthesis build_args = config["build_args"] - try: data = hls_model.build(**build_args.get(backend, {})) except Exception as e: pytest.skip(f"hls_model.build failed: {e}") + # Save synthesis report save_report(data, f"synthesis_report_{baseline_file_name}") - assert data and {'CSynthesisReport'}.issubset( + # Check synthesis report keys + expected_keys = EXPECTED_REPORT_KEYS.get(backend, set()) + assert data and expected_keys.issubset( data.keys() - ), "Synthesis failed: Missing expected keys in the synthesis report" + ), f"Synthesis failed: Missing expected keys in synthesis report: expected {expected_keys}, got {set(data.keys())}" + # Load baseline report version = config["tools_version"].get(backend) baseline_path = get_baseline_path(baseline_file_name, backend, version) - try: with open(baseline_path) as fp: baseline = json.load(fp) except FileNotFoundError: pytest.skip(f"Baseline file '{baseline_path}' not found.") - compare_reports_with_tolerance(data, baseline) + # Compare report against baseline using backend-specific rules + compare_func = COMPARE_FUNCS.get(backend) + if compare_func is None: + raise AssertionError(f"No comparison function defined for backend: {backend}") + + compare_func(data, baseline) diff --git a/test/pytest/test_keras_api.py b/test/pytest/test_keras_api.py index 772a7bcaed..d11ab7e1a6 100644 --- a/test/pytest/test_keras_api.py +++ b/test/pytest/test_keras_api.py @@ -4,7 +4,7 @@ import numpy as np import pytest import tensorflow as tf -from synthesis_helpers import test_synthesis +from synthesis_helpers import run_synthesis_test from tensorflow.keras.layers import ( ELU, Activation, @@ -76,7 +76,7 @@ def test_dense(backend, io_type, synthesis_config): assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # TODO: add ThresholdedReLU test when it can be made to pass @@ -120,7 +120,7 @@ def test_activations(activation_function, backend, io_type, synthesis_config): assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) padds_options = ['same', 'valid'] @@ -206,7 +206,7 @@ def test_conv1d(padds, backend, strategy, io_type, synthesis_config): assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) chans_options = ['channels_last'] @@ -330,7 +330,7 @@ def test_conv2d(chans, padds, backend, strategy, io_type, synthesis_config): assert list(hls_model.get_layers())[1].attributes['pad_left'] == 0 assert list(hls_model.get_layers())[1].attributes['pad_right'] == 0 - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # Currently only Vivado and Vitis is supported for io_stream. @@ -362,7 +362,7 @@ def test_depthwise2d(backend, io_type, synthesis_config): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) # Currently only Vivado and Vitis is supported for io_stream. @@ -392,7 +392,7 @@ def test_depthwise1d(backend, io_type, synthesis_config): np.testing.assert_allclose(y_qkeras, y_hls4ml.reshape(y_qkeras.shape), rtol=1e-2, atol=0.01) - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) pooling_layers = [MaxPooling1D, MaxPooling2D, AveragePooling1D, AveragePooling2D] @@ -524,4 +524,4 @@ def test_pooling(pooling, padds, chans, backend, synthesis_config): assert hls_pool.attributes['pad_left'] == 0 assert hls_pool.attributes['pad_right'] == 0 - test_synthesis(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From c51c1809d8df260b7a3f7d414c73a124a505ce11 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 11 Apr 2025 15:56:48 -0500 Subject: [PATCH 17/35] add temp test file --- test/pytest/test_keras_api_temp.py | 114 +++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 test/pytest/test_keras_api_temp.py diff --git a/test/pytest/test_keras_api_temp.py b/test/pytest/test_keras_api_temp.py new file mode 100644 index 0000000000..001a695d24 --- /dev/null +++ b/test/pytest/test_keras_api_temp.py @@ -0,0 +1,114 @@ +from pathlib import Path + +import numpy as np +import pytest +import tensorflow as tf +from synthesis_helpers import run_synthesis_test +from tensorflow.keras.layers import ( + ELU, + Activation, + Dense, + LeakyReLU, + PReLU, +) + +import hls4ml + +test_root_path = Path(__file__).parent + + +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_dense(backend, io_type, synthesis_config): + model = tf.keras.models.Sequential() + model.add( + Dense( + 2, + input_shape=(1,), + name='Dense', + use_bias=True, + kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10), + bias_initializer='zeros', + kernel_regularizer=None, + bias_regularizer=None, + activity_regularizer=None, + kernel_constraint=None, + bias_constraint=None, + ) + ) + model.add(Activation(activation='elu', name='Activation')) + model.compile(optimizer='adam', loss='mse') + + X_input = np.random.rand(100, 1) + + keras_prediction = model.predict(X_input) + + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}.json' + + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + + hls_model.compile() + + hls_prediction = hls_model.predict(X_input) + + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) + + assert len(model.layers) + 1 == len(hls_model.get_layers()) + assert list(hls_model.get_layers())[0].attributes['class_name'] == "InputLayer" + assert list(hls_model.get_layers())[1].attributes["class_name"] == model.layers[0]._name + assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' + assert list(hls_model.get_layers())[0].attributes['input_shape'] == list(model.layers[0].input_shape[1:]) + assert list(hls_model.get_layers())[1].attributes['n_in'] == model.layers[0].input_shape[1:][0] + assert list(hls_model.get_layers())[1].attributes['n_out'] == model.layers[0].output_shape[1:][0] + assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] + assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] + + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) + + +# TODO: add ThresholdedReLU test when it can be made to pass +# https://github.com/fastmachinelearning/hls4ml/issues/376 +@pytest.mark.parametrize( + "activation_function", + [ + Activation(activation='relu', name='relu'), + LeakyReLU(alpha=1.0), + ELU(alpha=1.0), + PReLU( + alpha_initializer="zeros", + ), + Activation(activation='sigmoid', name='sigmoid'), + ], +) +# ThresholdedReLU(theta=1.0)]) +@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) +@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) +def test_activations(activation_function, backend, io_type, synthesis_config): + model = tf.keras.models.Sequential() + model.add(Dense(64, input_shape=(1,), name='Dense', kernel_initializer='lecun_uniform', kernel_regularizer=None)) + model.add(activation_function) + + model.compile(optimizer='adam', loss='mse') + X_input = np.random.rand(100, 1) + keras_prediction = model.predict(X_input) + config = hls4ml.utils.config_from_keras_model(model) + output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') + baseline_file_name = f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}.json' + + hls_model = hls4ml.converters.convert_from_keras_model( + model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type + ) + hls_model.compile() + hls_prediction = hls_model.predict(X_input) + + np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) + + assert len(model.layers) + 1 == len(hls_model.get_layers()) + + assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ + + run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From e423d1694f8b37f8a5335c39be3c545c2fd762da Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 11 Apr 2025 15:58:17 -0500 Subject: [PATCH 18/35] update generate_ci_template.py --- test/pytest/generate_ci_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index 74c8ef6957..4946c82f32 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -7,10 +7,10 @@ template = """ -pytest.test_cvmfs_mount: +pytest.test_synthesis: extends: .pytest variables: - PYTESTFILE: test_keras_api.py + PYTESTFILE: test_keras_api_temp.py EXAMPLEMODEL: 0 """ From d9c613a1dc698748430ce7ef6c89c03b559868cb Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 11 Apr 2025 16:19:56 -0500 Subject: [PATCH 19/35] update temp test file --- test/pytest/test_keras_api_temp.py | 52 +----------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/test/pytest/test_keras_api_temp.py b/test/pytest/test_keras_api_temp.py index 001a695d24..cc7582a176 100644 --- a/test/pytest/test_keras_api_temp.py +++ b/test/pytest/test_keras_api_temp.py @@ -4,13 +4,7 @@ import pytest import tensorflow as tf from synthesis_helpers import run_synthesis_test -from tensorflow.keras.layers import ( - ELU, - Activation, - Dense, - LeakyReLU, - PReLU, -) +from tensorflow.keras.layers import Activation, Dense import hls4ml @@ -68,47 +62,3 @@ def test_dense(backend, io_type, synthesis_config): assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) - - -# TODO: add ThresholdedReLU test when it can be made to pass -# https://github.com/fastmachinelearning/hls4ml/issues/376 -@pytest.mark.parametrize( - "activation_function", - [ - Activation(activation='relu', name='relu'), - LeakyReLU(alpha=1.0), - ELU(alpha=1.0), - PReLU( - alpha_initializer="zeros", - ), - Activation(activation='sigmoid', name='sigmoid'), - ], -) -# ThresholdedReLU(theta=1.0)]) -@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) -@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_activations(activation_function, backend, io_type, synthesis_config): - model = tf.keras.models.Sequential() - model.add(Dense(64, input_shape=(1,), name='Dense', kernel_initializer='lecun_uniform', kernel_regularizer=None)) - model.add(activation_function) - - model.compile(optimizer='adam', loss='mse') - X_input = np.random.rand(100, 1) - keras_prediction = model.predict(X_input) - config = hls4ml.utils.config_from_keras_model(model) - output_dir = str(test_root_path / f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_activations_{activation_function.name}_{backend}_{io_type}.json' - - hls_model = hls4ml.converters.convert_from_keras_model( - model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type - ) - hls_model.compile() - hls_prediction = hls_model.predict(X_input) - - np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) - - assert len(model.layers) + 1 == len(hls_model.get_layers()) - - assert list(hls_model.get_layers())[2].attributes['class_name'] == activation_function.__class__.__name__ - - run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From 97f3fe6c79ac7cd8cf6fb2097a28a4bcf1ab7689 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 11 Apr 2025 16:39:23 -0500 Subject: [PATCH 20/35] install libidn --- test/pytest/ci-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 08e398ff29..e0e4fb9cc2 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -12,6 +12,7 @@ - git submodule update --init --recursive test/pytest/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] + - sudo yum install -y libidn - sudo yum install libtinfo.so.6 -y - sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx From 1ed33b44631b82fda5f3a6d5a8544f3a90a49a7f Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 14 Apr 2025 15:22:44 -0500 Subject: [PATCH 21/35] test cmake version --- test/pytest/ci-template.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index e0e4fb9cc2..d3d971d4bb 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -12,12 +12,14 @@ - git submodule update --init --recursive test/pytest/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] - - sudo yum install -y libidn - sudo yum install libtinfo.so.6 -y - sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx - source /opt/Xilinx/Vivado/2020.1/settings64.sh - export RUN_SYNTHESIS=true + - which cmake + - cmake --version + - ldd $(which cmake) script: - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed From d39d7dc152f8c17de2b9e0654fc8ad33d45d2421 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 14 Apr 2025 15:40:04 -0500 Subject: [PATCH 22/35] fix cmake error --- test/pytest/ci-template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index d3d971d4bb..c22b0d7e0b 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -17,6 +17,7 @@ - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx - source /opt/Xilinx/Vivado/2020.1/settings64.sh - export RUN_SYNTHESIS=true + - export PATH=/usr/local/bin - which cmake - cmake --version - ldd $(which cmake) From 1280aa744e82f389a8276370af27e21c62d67043 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 14 Apr 2025 15:53:52 -0500 Subject: [PATCH 23/35] fix cmake error --- test/pytest/ci-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index c22b0d7e0b..6e97833dd8 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -17,7 +17,7 @@ - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx - source /opt/Xilinx/Vivado/2020.1/settings64.sh - export RUN_SYNTHESIS=true - - export PATH=/usr/local/bin + - export PATH=/usr/bin:$PATH - which cmake - cmake --version - ldd $(which cmake) From aaf865f0c4038c65a9bec121ad182ca4c23d444a Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Mon, 14 Apr 2025 16:46:56 -0500 Subject: [PATCH 24/35] update baselines, clean ci-template --- test/pytest/baselines | 2 +- test/pytest/ci-template.yml | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/test/pytest/baselines b/test/pytest/baselines index 5d00fbdda2..18144e1379 160000 --- a/test/pytest/baselines +++ b/test/pytest/baselines @@ -1 +1 @@ -Subproject commit 5d00fbdda2d9209c8da53d6674b7a8f1ba7ae113 +Subproject commit 18144e13796333a271029ba9dc46b570256d3157 diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 6e97833dd8..6eb430a8a7 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -18,9 +18,6 @@ - source /opt/Xilinx/Vivado/2020.1/settings64.sh - export RUN_SYNTHESIS=true - export PATH=/usr/bin:$PATH - - which cmake - - cmake --version - - ldd $(which cmake) script: - cd test/pytest - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed From db21ed4471c895e9d3f3a39ac959e4cf736b9854 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 15 Apr 2025 12:50:16 -0500 Subject: [PATCH 25/35] remove test_keras_api_temp.py --- test/pytest/generate_ci_yaml.py | 72 +++++++++++++++++++++++++++--- test/pytest/test_keras_api_temp.py | 64 -------------------------- 2 files changed, 65 insertions(+), 71 deletions(-) delete mode 100644 test/pytest/test_keras_api_temp.py diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index 4946c82f32..adc3d680ab 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -1,3 +1,7 @@ +import itertools +import os +from pathlib import Path + import yaml ''' @@ -7,20 +11,74 @@ template = """ -pytest.test_synthesis: +pytest.{}: extends: .pytest variables: - PYTESTFILE: test_keras_api_temp.py - EXAMPLEMODEL: 0 + PYTESTFILE: {} + EXAMPLEMODEL: {} """ -def generate_fixed_test_yaml(): - yml = yaml.safe_load(template) +n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4)) + +# Blacklisted tests will be skipped +BLACKLIST = {'test_reduction'} + +# Long-running tests will not be bundled with other tests +LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} + + +def path_to_name(test_path): + path = Path(test_path) + name = path.stem.replace('test_', '') + return name + + +def batched(iterable, chunk_size): + iterator = iter(iterable) + while chunk := tuple(itertools.islice(iterator, chunk_size)): + yield chunk + + +def uses_example_model(test_filename): + with open(test_filename) as f: + content = f.read() + return 'example-models' in content + + +def generate_test_yaml(test_root='.'): + test_root = Path(test_root) + test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem not in (BLACKLIST | LONGLIST)] + need_example_models = [uses_example_model(path) for path in test_paths] + + idxs = list(range(len(need_example_models))) + idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}') + + yml = None + for batch_idxs in batched(idxs, n_test_files_per_yml): + batch_paths: list[Path] = [test_paths[i] for i in batch_idxs] + names = [path_to_name(path) for path in batch_paths] + name = '+'.join(names) + test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths]) + batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs])) + diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model)) + if yml is None: + yml = diff_yml + else: + yml.update(diff_yml) + + test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST] + for path in test_paths: + name = path.stem.replace('test_', '') + test_file = str(path.relative_to(test_root)) + needs_examples = uses_example_model(path) + diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples))) + yml.update(diff_yml) + return yml if __name__ == '__main__': - yml = generate_fixed_test_yaml() + yml = generate_test_yaml(Path(__file__).parent) with open('pytests.yml', 'w') as yamlfile: - yaml.safe_dump(yml, yamlfile, default_flow_style=False) + yaml.safe_dump(yml, yamlfile) diff --git a/test/pytest/test_keras_api_temp.py b/test/pytest/test_keras_api_temp.py deleted file mode 100644 index cc7582a176..0000000000 --- a/test/pytest/test_keras_api_temp.py +++ /dev/null @@ -1,64 +0,0 @@ -from pathlib import Path - -import numpy as np -import pytest -import tensorflow as tf -from synthesis_helpers import run_synthesis_test -from tensorflow.keras.layers import Activation, Dense - -import hls4ml - -test_root_path = Path(__file__).parent - - -@pytest.mark.parametrize('backend', ['Vivado', 'Vitis', 'Quartus', 'oneAPI']) -@pytest.mark.parametrize('io_type', ['io_parallel', 'io_stream']) -def test_dense(backend, io_type, synthesis_config): - model = tf.keras.models.Sequential() - model.add( - Dense( - 2, - input_shape=(1,), - name='Dense', - use_bias=True, - kernel_initializer=tf.keras.initializers.RandomUniform(minval=1, maxval=10), - bias_initializer='zeros', - kernel_regularizer=None, - bias_regularizer=None, - activity_regularizer=None, - kernel_constraint=None, - bias_constraint=None, - ) - ) - model.add(Activation(activation='elu', name='Activation')) - model.compile(optimizer='adam', loss='mse') - - X_input = np.random.rand(100, 1) - - keras_prediction = model.predict(X_input) - - config = hls4ml.utils.config_from_keras_model(model) - output_dir = str(test_root_path / f'hls4mlprj_keras_api_dense_{backend}_{io_type}') - baseline_file_name = f'hls4mlprj_keras_api_dense_{backend}_{io_type}.json' - - hls_model = hls4ml.converters.convert_from_keras_model( - model, hls_config=config, output_dir=output_dir, backend=backend, io_type=io_type - ) - - hls_model.compile() - - hls_prediction = hls_model.predict(X_input) - - np.testing.assert_allclose(hls_prediction, keras_prediction, rtol=1e-2, atol=0.01) - - assert len(model.layers) + 1 == len(hls_model.get_layers()) - assert list(hls_model.get_layers())[0].attributes['class_name'] == "InputLayer" - assert list(hls_model.get_layers())[1].attributes["class_name"] == model.layers[0]._name - assert list(hls_model.get_layers())[2].attributes['class_name'] == 'ELU' - assert list(hls_model.get_layers())[0].attributes['input_shape'] == list(model.layers[0].input_shape[1:]) - assert list(hls_model.get_layers())[1].attributes['n_in'] == model.layers[0].input_shape[1:][0] - assert list(hls_model.get_layers())[1].attributes['n_out'] == model.layers[0].output_shape[1:][0] - assert list(hls_model.get_layers())[2].attributes['activation'] == str(model.layers[1].activation).split()[1] - assert list(hls_model.get_layers())[1].attributes['activation'] == str(model.layers[0].activation).split()[1] - - run_synthesis_test(config=synthesis_config, hls_model=hls_model, baseline_file_name=baseline_file_name, backend=backend) From 45f862b596d365c929940139e8beee4028a63f78 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 17 Jun 2025 13:52:25 -0500 Subject: [PATCH 26/35] separate test_keras_api in a single job --- test/pytest/generate_ci_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index adc3d680ab..f65bedc2b4 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -25,7 +25,7 @@ BLACKLIST = {'test_reduction'} # Long-running tests will not be bundled with other tests -LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} +LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api', 'test_keras_api'} def path_to_name(test_path): From 37d184f9238e9e73033e6bd1316520424e4a464b Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 17 Jun 2025 16:35:03 -0500 Subject: [PATCH 27/35] update genereate_ci_yaml.py to batch syntehsis tests --- test/pytest/generate_ci_yaml.py | 57 ++++++++++++++++++++++++++++----- 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index f65bedc2b4..537e0ea6e2 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -1,5 +1,6 @@ import itertools import os +import subprocess from pathlib import Path import yaml @@ -25,7 +26,27 @@ BLACKLIST = {'test_reduction'} # Long-running tests will not be bundled with other tests -LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api', 'test_keras_api'} +LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} + +# Test files to split by individual test cases (stem only, no .py) +# Value = chunk size per CI job +SPLIT_BY_TEST_CASE = { + 'test_keras_api': 20, +} + + +def collect_test_cases(test_file): + result = subprocess.run(['pytest', '--collect-only', '-q', str(test_file)], capture_output=True, text=True) + + lines = result.stdout.splitlines() + test_ids = [line.strip().split('/')[-1] for line in lines if "::" in line] # get only filename + nodeid + return test_ids + + +def batched(iterable, batch_size): + it = iter(iterable) + while batch := list(itertools.islice(it, batch_size)): + yield batch def path_to_name(test_path): @@ -34,12 +55,6 @@ def path_to_name(test_path): return name -def batched(iterable, chunk_size): - iterator = iter(iterable) - while chunk := tuple(itertools.islice(iterator, chunk_size)): - yield chunk - - def uses_example_model(test_filename): with open(test_filename) as f: content = f.read() @@ -48,7 +63,11 @@ def uses_example_model(test_filename): def generate_test_yaml(test_root='.'): test_root = Path(test_root) - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem not in (BLACKLIST | LONGLIST)] + test_paths = [ + path + for path in test_root.glob('**/test_*.py') + if path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys())) + ] need_example_models = [uses_example_model(path) for path in test_paths] idxs = list(range(len(need_example_models))) @@ -75,6 +94,28 @@ def generate_test_yaml(test_root='.'): diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples))) yml.update(diff_yml) + # Handle split-by-test-case files + test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE] + for path in test_paths: + stem = path.stem + name_base = stem.replace('test_', '') + test_file = str(path.relative_to(test_root)) + test_ids = collect_test_cases(path) + chunk_size = SPLIT_BY_TEST_CASE[stem] + needs_examples = uses_example_model(path) + + for i, batch in enumerate(batched(test_ids, chunk_size)): + job_name = f"{name_base}_part{i}" + batch_ids = " ".join(batch).strip().replace("\n", " ") # flat single-line string + job_key = f"pytest.{job_name}" + job_entry = { + job_key: {"extends": ".pytest", "variables": {"PYTESTFILE": batch_ids, "EXAMPLEMODEL": int(needs_examples)}} + } + if yml is None: + yml = job_entry + else: + yml.update(job_entry) + return yml From 44a11a096104ab0f47932044ebcbda53b925e992 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 17 Jun 2025 16:40:09 -0500 Subject: [PATCH 28/35] update gitlab-ci.yml to support pytest --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 89535c1937..b0baf12f82 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ generator: tags: - k8s-default before_script: - - pip install pyyaml + - pip install pyyaml pytest script: - cd test/pytest - python generate_ci_yaml.py From df6a4871548a8a11eb16287586d5e32d94127f70 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 18 Jun 2025 15:45:59 -0500 Subject: [PATCH 29/35] create jobs for single test functions --- .gitlab-ci.yml | 2 +- test/pytest/generate_ci_yaml.py | 32 ++++++++++++++++---------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b0baf12f82..89535c1937 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ generator: tags: - k8s-default before_script: - - pip install pyyaml pytest + - pip install pyyaml script: - cd test/pytest - python generate_ci_yaml.py diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index 537e0ea6e2..f9154b7d40 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -1,6 +1,6 @@ +import ast import itertools import os -import subprocess from pathlib import Path import yaml @@ -31,16 +31,20 @@ # Test files to split by individual test cases (stem only, no .py) # Value = chunk size per CI job SPLIT_BY_TEST_CASE = { - 'test_keras_api': 20, + 'test_keras_api': 1, } -def collect_test_cases(test_file): - result = subprocess.run(['pytest', '--collect-only', '-q', str(test_file)], capture_output=True, text=True) +def collect_test_functions_from_ast(test_file): + """Collect all test function names using AST parsing (no imports).""" + with open(test_file, encoding='utf-8') as f: + tree = ast.parse(f.read(), filename=str(test_file)) - lines = result.stdout.splitlines() - test_ids = [line.strip().split('/')[-1] for line in lines if "::" in line] # get only filename + nodeid - return test_ids + test_funcs = [] + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name.startswith("test"): + test_funcs.append(f"{test_file}::{node.name}") + return test_funcs def batched(iterable, batch_size): @@ -94,27 +98,23 @@ def generate_test_yaml(test_root='.'): diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples))) yml.update(diff_yml) - # Handle split-by-test-case files test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE] for path in test_paths: stem = path.stem name_base = stem.replace('test_', '') test_file = str(path.relative_to(test_root)) - test_ids = collect_test_cases(path) + test_ids = collect_test_functions_from_ast(test_file) chunk_size = SPLIT_BY_TEST_CASE[stem] needs_examples = uses_example_model(path) for i, batch in enumerate(batched(test_ids, chunk_size)): job_name = f"{name_base}_part{i}" - batch_ids = " ".join(batch).strip().replace("\n", " ") # flat single-line string - job_key = f"pytest.{job_name}" - job_entry = { - job_key: {"extends": ".pytest", "variables": {"PYTESTFILE": batch_ids, "EXAMPLEMODEL": int(needs_examples)}} - } + test_file_args = " ".join(batch).strip().replace("\n", " ") + diff_yml = yaml.safe_load(template.format(job_name, test_file_args, int(needs_examples))) if yml is None: - yml = job_entry + yml = diff_yml else: - yml.update(job_entry) + yml.update(diff_yml) return yml From f7c0c79c61d1372ba6b0435f38337e56971e3613 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Wed, 18 Jun 2025 16:13:07 -0500 Subject: [PATCH 30/35] update test policy from skip to fail --- test/pytest/synthesis_helpers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/synthesis_helpers.py b/test/pytest/synthesis_helpers.py index 02de79ab97..280ed63458 100644 --- a/test/pytest/synthesis_helpers.py +++ b/test/pytest/synthesis_helpers.py @@ -151,7 +151,7 @@ def run_synthesis_test(config, hls_model, baseline_file_name, backend): try: data = hls_model.build(**build_args.get(backend, {})) except Exception as e: - pytest.skip(f"hls_model.build failed: {e}") + pytest.fail(f"hls_model.build failed: {e}") # Save synthesis report save_report(data, f"synthesis_report_{baseline_file_name}") @@ -169,7 +169,7 @@ def run_synthesis_test(config, hls_model, baseline_file_name, backend): with open(baseline_path) as fp: baseline = json.load(fp) except FileNotFoundError: - pytest.skip(f"Baseline file '{baseline_path}' not found.") + pytest.fail(f"Baseline file '{baseline_path}' not found.") # Compare report against baseline using backend-specific rules compare_func = COMPARE_FUNCS.get(backend) From 0a90de3bb58805a931091a50447c2f0b8c9d18cb Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Thu, 19 Jun 2025 11:34:26 -0500 Subject: [PATCH 31/35] test conv1d vivado --- test/pytest/ci-template.yml | 2 +- test/pytest/generate_ci_yaml.py | 243 ++++++++++++++++++-------------- 2 files changed, 135 insertions(+), 110 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 6eb430a8a7..51c0462c0a 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -20,7 +20,7 @@ - export PATH=/usr/bin:$PATH script: - cd test/pytest - - pytest $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed + - pytest -s $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed artifacts: when: always reports: diff --git a/test/pytest/generate_ci_yaml.py b/test/pytest/generate_ci_yaml.py index f9154b7d40..9e4914d78e 100644 --- a/test/pytest/generate_ci_yaml.py +++ b/test/pytest/generate_ci_yaml.py @@ -1,125 +1,150 @@ -import ast -import itertools -import os -from pathlib import Path - import yaml ''' -Create a Gitlab CI yml file with a separate entry for each test_* file -in the pytests directory to parallelise the CI jobs. +Minimal GitLab CI yml generator for testing purposes. +Generates a single job entry for test_keras_api.py::test_activations. ''' - template = """ -pytest.{}: +pytest.keras_api_part2: extends: .pytest variables: - PYTESTFILE: {} - EXAMPLEMODEL: {} + PYTESTFILE: test_keras_api.py::test_conv1d + EXAMPLEMODEL: 0 """ -n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4)) - -# Blacklisted tests will be skipped -BLACKLIST = {'test_reduction'} - -# Long-running tests will not be bundled with other tests -LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} - -# Test files to split by individual test cases (stem only, no .py) -# Value = chunk size per CI job -SPLIT_BY_TEST_CASE = { - 'test_keras_api': 1, -} - - -def collect_test_functions_from_ast(test_file): - """Collect all test function names using AST parsing (no imports).""" - with open(test_file, encoding='utf-8') as f: - tree = ast.parse(f.read(), filename=str(test_file)) - - test_funcs = [] - for node in tree.body: - if isinstance(node, ast.FunctionDef) and node.name.startswith("test"): - test_funcs.append(f"{test_file}::{node.name}") - return test_funcs - - -def batched(iterable, batch_size): - it = iter(iterable) - while batch := list(itertools.islice(it, batch_size)): - yield batch - - -def path_to_name(test_path): - path = Path(test_path) - name = path.stem.replace('test_', '') - return name - - -def uses_example_model(test_filename): - with open(test_filename) as f: - content = f.read() - return 'example-models' in content - - -def generate_test_yaml(test_root='.'): - test_root = Path(test_root) - test_paths = [ - path - for path in test_root.glob('**/test_*.py') - if path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys())) - ] - need_example_models = [uses_example_model(path) for path in test_paths] - - idxs = list(range(len(need_example_models))) - idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}') - - yml = None - for batch_idxs in batched(idxs, n_test_files_per_yml): - batch_paths: list[Path] = [test_paths[i] for i in batch_idxs] - names = [path_to_name(path) for path in batch_paths] - name = '+'.join(names) - test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths]) - batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs])) - diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model)) - if yml is None: - yml = diff_yml - else: - yml.update(diff_yml) - - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST] - for path in test_paths: - name = path.stem.replace('test_', '') - test_file = str(path.relative_to(test_root)) - needs_examples = uses_example_model(path) - diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples))) - yml.update(diff_yml) - - test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE] - for path in test_paths: - stem = path.stem - name_base = stem.replace('test_', '') - test_file = str(path.relative_to(test_root)) - test_ids = collect_test_functions_from_ast(test_file) - chunk_size = SPLIT_BY_TEST_CASE[stem] - needs_examples = uses_example_model(path) - - for i, batch in enumerate(batched(test_ids, chunk_size)): - job_name = f"{name_base}_part{i}" - test_file_args = " ".join(batch).strip().replace("\n", " ") - diff_yml = yaml.safe_load(template.format(job_name, test_file_args, int(needs_examples))) - if yml is None: - yml = diff_yml - else: - yml.update(diff_yml) - - return yml +def generate_test_yaml(): + return yaml.safe_load(template) if __name__ == '__main__': - yml = generate_test_yaml(Path(__file__).parent) + yml = generate_test_yaml() with open('pytests.yml', 'w') as yamlfile: yaml.safe_dump(yml, yamlfile) + +# import ast +# import itertools +# import os +# from pathlib import Path + +# import yaml + +# ''' +# Create a Gitlab CI yml file with a separate entry for each test_* file +# in the pytests directory to parallelise the CI jobs. +# ''' + + +# template = """ +# pytest.{}: +# extends: .pytest +# variables: +# PYTESTFILE: {} +# EXAMPLEMODEL: {} +# """ + + +# n_test_files_per_yml = int(os.environ.get('N_TESTS_PER_YAML', 4)) + +# # Blacklisted tests will be skipped +# BLACKLIST = {'test_reduction'} + +# # Long-running tests will not be bundled with other tests +# LONGLIST = {'test_hgq_layers', 'test_hgq_players', 'test_qkeras', 'test_pytorch_api'} + +# # Test files to split by individual test cases (stem only, no .py) +# # Value = chunk size per CI job +# SPLIT_BY_TEST_CASE = { +# 'test_keras_api': 1, +# } + + +# def collect_test_functions_from_ast(test_file): +# """Collect all test function names using AST parsing (no imports).""" +# with open(test_file, encoding='utf-8') as f: +# tree = ast.parse(f.read(), filename=str(test_file)) + +# test_funcs = [] +# for node in tree.body: +# if isinstance(node, ast.FunctionDef) and node.name.startswith("test"): +# test_funcs.append(f"{test_file}::{node.name}") +# return test_funcs + + +# def batched(iterable, batch_size): +# it = iter(iterable) +# while batch := list(itertools.islice(it, batch_size)): +# yield batch + + +# def path_to_name(test_path): +# path = Path(test_path) +# name = path.stem.replace('test_', '') +# return name + + +# def uses_example_model(test_filename): +# with open(test_filename) as f: +# content = f.read() +# return 'example-models' in content + + +# def generate_test_yaml(test_root='.'): +# test_root = Path(test_root) +# test_paths = [ +# path +# for path in test_root.glob('**/test_*.py') +# if path.stem not in (BLACKLIST | LONGLIST | set(SPLIT_BY_TEST_CASE.keys())) +# ] +# need_example_models = [uses_example_model(path) for path in test_paths] + +# idxs = list(range(len(need_example_models))) +# idxs = sorted(idxs, key=lambda i: f'{need_example_models[i]}_{path_to_name(test_paths[i])}') + +# yml = None +# for batch_idxs in batched(idxs, n_test_files_per_yml): +# batch_paths: list[Path] = [test_paths[i] for i in batch_idxs] +# names = [path_to_name(path) for path in batch_paths] +# name = '+'.join(names) +# test_files = ' '.join([str(path.relative_to(test_root)) for path in batch_paths]) +# batch_need_example_model = int(any([need_example_models[i] for i in batch_idxs])) +# diff_yml = yaml.safe_load(template.format(name, test_files, batch_need_example_model)) +# if yml is None: +# yml = diff_yml +# else: +# yml.update(diff_yml) + +# test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in LONGLIST] +# for path in test_paths: +# name = path.stem.replace('test_', '') +# test_file = str(path.relative_to(test_root)) +# needs_examples = uses_example_model(path) +# diff_yml = yaml.safe_load(template.format(name, test_file, int(needs_examples))) +# yml.update(diff_yml) + +# test_paths = [path for path in test_root.glob('**/test_*.py') if path.stem in SPLIT_BY_TEST_CASE] +# for path in test_paths: +# stem = path.stem +# name_base = stem.replace('test_', '') +# test_file = str(path.relative_to(test_root)) +# test_ids = collect_test_functions_from_ast(test_file) +# chunk_size = SPLIT_BY_TEST_CASE[stem] +# needs_examples = uses_example_model(path) + +# for i, batch in enumerate(batched(test_ids, chunk_size)): +# job_name = f"{name_base}_part{i}" +# test_file_args = " ".join(batch).strip().replace("\n", " ") +# diff_yml = yaml.safe_load(template.format(job_name, test_file_args, int(needs_examples))) +# if yml is None: +# yml = diff_yml +# else: +# yml.update(diff_yml) + +# return yml + + +# if __name__ == '__main__': +# yml = generate_test_yaml(Path(__file__).parent) +# with open('pytests.yml', 'w') as yamlfile: +# yaml.safe_dump(yml, yamlfile) From 8a3f0ae92254fb8137f35e59d5c27d27f0c1a385 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 20 Jun 2025 13:29:57 -0500 Subject: [PATCH 32/35] use apptainer in cvmfs --- test/pytest/ci-template.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 51c0462c0a..1e1310a172 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -12,12 +12,28 @@ - git submodule update --init --recursive test/pytest/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] - - sudo yum install libtinfo.so.6 -y - - sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 - - sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx - - source /opt/Xilinx/Vivado/2020.1/settings64.sh + - sudo yum install -y epel-release + - sudo yum install -y apptainer squashfs-tools fuse-overlayfs + #- sudo yum install libtinfo.so.6 -y + #- sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 + #- sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx + #- source /opt/Xilinx/Vivado/2020.1/settings64.sh - export RUN_SYNTHESIS=true - - export PATH=/usr/bin:$PATH + #- export PATH=/usr/bin:$PATH + + - export TOOL_VERSION=2020.1 + + - mkdir -p cmd_vivado_${TOOL_VERSION} + - echo '#!/bin/bash' > cmd_vivado_${TOOL_VERSION}/vivado_hls + - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/ vivado_hls \"\$@\"" >> cmd_vivado_${TOOL_VERSION}/vivado_hls + - chmod +x cmd_vivado_${TOOL_VERSION}/vivado_hls + + # Add it to PATH + - export PATH=$PWD/cmd_vivado_${TOOL_VERSION}:$PATH + + - which vivado_hls + - vivado_hls -version || echo "Warning: unable to detect Vivado version" + script: - cd test/pytest - pytest -s $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed From 9581f274d43dd7811dbcda5edbe053dd3233fd07 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 20 Jun 2025 13:35:20 -0500 Subject: [PATCH 33/35] fix bug ci-tempalte --- test/pytest/ci-template.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 1e1310a172..2ca1987a7d 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -14,26 +14,15 @@ - pip install .[testing,sr,optimization] - sudo yum install -y epel-release - sudo yum install -y apptainer squashfs-tools fuse-overlayfs - #- sudo yum install libtinfo.so.6 -y - #- sudo ln -s /lib64/libtinfo.so.6 /lib64/libtinfo.so.5 - #- sudo ln -s /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/opt/Xilinx /opt/Xilinx - #- source /opt/Xilinx/Vivado/2020.1/settings64.sh - export RUN_SYNTHESIS=true - #- export PATH=/usr/bin:$PATH - - export TOOL_VERSION=2020.1 - - mkdir -p cmd_vivado_${TOOL_VERSION} - echo '#!/bin/bash' > cmd_vivado_${TOOL_VERSION}/vivado_hls - echo "apptainer exec /cvmfs/projects.cern.ch/hls4ml/vivado/2020.1_v1/vivado-2020.1_v1/ vivado_hls \"\$@\"" >> cmd_vivado_${TOOL_VERSION}/vivado_hls - chmod +x cmd_vivado_${TOOL_VERSION}/vivado_hls - - # Add it to PATH - export PATH=$PWD/cmd_vivado_${TOOL_VERSION}:$PATH - - which vivado_hls - - vivado_hls -version || echo "Warning: unable to detect Vivado version" - + - vivado_hls -version script: - cd test/pytest - pytest -s $PYTESTFILE -rA --cov-report xml --cov-report term --cov=hls4ml --junitxml=report.xml --randomly-seed=42 --randomly-dont-reorganize --randomly-dont-reset-seed From c2f4297dfd2deaac9c1571bc2ef065cc0dc28840 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Fri, 20 Jun 2025 16:11:11 -0500 Subject: [PATCH 34/35] use apptainer-suid --- test/pytest/ci-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index 2ca1987a7d..d15fe3fb0b 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -13,7 +13,7 @@ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] - sudo yum install -y epel-release - - sudo yum install -y apptainer squashfs-tools fuse-overlayfs + - sudo yum install -y apptainer-suid squashfs-tools fuse-overlayfs - export RUN_SYNTHESIS=true - export TOOL_VERSION=2020.1 - mkdir -p cmd_vivado_${TOOL_VERSION} From 2872ce7841776fbafefecbd841755bfdffe3ff71 Mon Sep 17 00:00:00 2001 From: Marco Colombo Date: Tue, 24 Jun 2025 11:07:55 -0500 Subject: [PATCH 35/35] check apptainer on host --- test/pytest/ci-template.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/pytest/ci-template.yml b/test/pytest/ci-template.yml index d15fe3fb0b..4dede94312 100644 --- a/test/pytest/ci-template.yml +++ b/test/pytest/ci-template.yml @@ -12,8 +12,6 @@ - git submodule update --init --recursive test/pytest/ - if [ $EXAMPLEMODEL == 1 ]; then git submodule update --init example-models; fi - pip install .[testing,sr,optimization] - - sudo yum install -y epel-release - - sudo yum install -y apptainer-suid squashfs-tools fuse-overlayfs - export RUN_SYNTHESIS=true - export TOOL_VERSION=2020.1 - mkdir -p cmd_vivado_${TOOL_VERSION}