From 1c6386c99f1dee6f718322f93e0316c0fcf613ff Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 11 Jun 2025 18:59:59 +0200 Subject: [PATCH 1/2] rework CI workflow for missing installations to only check whether eb_hooks.py is up-to-date (check for missing installations is done in EESSI/software-layer repo) --- .github/workflows/test-eb-hooks.yml | 58 ++++++ .github/workflows/test-software.eessi.io.yml | 179 ------------------- 2 files changed, 58 insertions(+), 179 deletions(-) create mode 100644 .github/workflows/test-eb-hooks.yml delete mode 100644 .github/workflows/test-software.eessi.io.yml diff --git a/.github/workflows/test-eb-hooks.yml b/.github/workflows/test-eb-hooks.yml new file mode 100644 index 00000000..aa54655a --- /dev/null +++ b/.github/workflows/test-eb-hooks.yml @@ -0,0 +1,58 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check whether eb_hooks.py script is up-to-date +on: + push: + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +env: + EESSI_VERSION: '2023.06' +jobs: + check_eb_hooks: + runs-on: ubuntu-24.04 + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + fetch-depth: 0 # Fetch all history for all branches and tags + + - name: Show host system info + run: | + echo "/proc/cpuinfo:" + cat /proc/cpuinfo + echo + echo "lscpu:" + lscpu + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + + - name: Check that EasyBuild hook is up to date + if: ${{ github.event_name == 'pull_request' }} + run: | + FILE="eb_hooks.py" + TEMP_FILE="$(mktemp)" + + # Fetch base branch + git fetch origin ${{ github.base_ref }} + + # Check if the hooks has changed in the PR + if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then + echo "Hooks changed in PR. Using PR version." + cp "$FILE" "$TEMP_FILE" + else + echo "File not changed in PR. Using default branch version." + git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE" + fi + + # Compare the hooks to what is shipped in the repository + # (it is overkill, but harmless, to do this for every architecture) + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash + module load EESSI-extend + diff "$TEMP_FILE" "$EASYBUILD_HOOKS" diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml deleted file mode 100644 index b517246b..00000000 --- a/.github/workflows/test-software.eessi.io.yml +++ /dev/null @@ -1,179 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Check for missing software installations in software.eessi.io -on: - push: - branches: [ "*-software.eessi.io" ] - pull_request: - workflow_dispatch: -permissions: - contents: read # to fetch code (actions/checkout) -env: - EESSI_VERSION: '2023.06' - EESSI_ACCELERATOR_TARGETS: | - x86_64/amd/zen2: - - nvidia/cc80 - x86_64/amd/zen3: - - nvidia/cc80 -jobs: - check_missing: - strategy: - fail-fast: false - matrix: - include: - # Arm CPU targets - - runs_on: ubuntu-24.04-arm - EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic - - runs_on: ubuntu-24.04-arm - EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_n1 - - runs_on: ubuntu-24.04-arm - EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_v1 - - runs_on: ubuntu-24.04-arm - EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/nvidia/grace - # x86_64 CPU targets - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen2 - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen3 - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen4 - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/haswell - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/sapphirerapids - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/skylake_avx512 - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/icelake - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/cascadelake - - runs_on: ubuntu-24.04 - EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic - runs-on: ${{ matrix.runs_on }} - steps: - - name: Check out software-layer repository - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 # Fetch all history for all branches and tags - - - name: Show host system info - run: | - echo "/proc/cpuinfo:" - cat /proc/cpuinfo - echo - echo "lscpu:" - lscpu - - - name: Mount EESSI CernVM-FS pilot repository - uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 - with: - cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb - cvmfs_http_proxy: DIRECT - cvmfs_repositories: software.eessi.io - - - name: Test check_missing_installations.sh script - run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash - # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), - # to prevent issues with checks in the Easybuild configuration that use this variable - export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION} - export EESSI_OS_TYPE=linux - env | grep ^EESSI | sort - - # first check the CPU-only builds for this CPU target - echo "first run check_missing_installations.sh for CPU-only builds" - for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} .github/workflows/scripts/only_latest_easystacks.sh); do - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g') - echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..." - module purge - module load EasyBuild/${eb_version} - which eb - eb --version - ./check_missing_installations.sh ${easystack_file} - ec=$? - if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi - done - - # now check the accelerator builds for this CPU target - accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") - if [ -z ${accelerators} ]; then - echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" - else - for accel in ${accelerators}; do - module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all - echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do - eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g') - echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..." - module purge - module load EasyBuild/${eb_version} - which eb - eb --version - ./check_missing_installations.sh ${easystack_file} - ec=$? - if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi - done - module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all - done - fi - - # make sure that Lmod cache file is present - ls -l ${EESSI_SOFTWARE_PATH}/.lmod/cache/spiderT.lua - - - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) - run: | - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash - # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), - # to prevent issues with checks in the Easybuild configuration that use this variable - export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} - module load EasyBuild - which eb - eb --version - export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION} - export EESSI_OS_TYPE=linux - env | grep ^EESSI | sort - # create dummy easystack file with a single entry (something that is not installed in EESSI) - easystack_file="test.yml" - echo "easyconfigs:" > ${easystack_file} - echo " - GCC-8.3.0:" >> ${easystack_file} - echo "created easystack file '${easystack_file}' with a missing installation (GCC/8.3.0):" - cat ${easystack_file} - # note, check_missing_installations.sh exits 1 if a package was - # missing, which is intepreted as false (exit code based, not - # boolean logic), hence when the script exits 0 if no package was - # missing it is interpreted as true, thus the test did not capture - # the missing package - if ./check_missing_installations.sh ${easystack_file}; then - echo "did NOT capture missing package; test FAILED" - exit 1 - else - echo "captured missing package; test PASSED" - exit 0 - fi - - - name: Check that EasyBuild hook is up to date - if: ${{ github.event_name == 'pull_request' }} - run: | - FILE="eb_hooks.py" - TEMP_FILE="$(mktemp)" - - # Fetch base branch - git fetch origin ${{ github.base_ref }} - - # Check if the hooks has changed in the PR - if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then - echo "Hooks changed in PR. Using PR version." - cp "$FILE" "$TEMP_FILE" - else - echo "File not changed in PR. Using default branch version." - git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE" - fi - - # Compare the hooks to what is shipped in the repository - # (it is overkill, but harmless, to do this for every architecture) - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} - source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash - module load EESSI-extend - diff "$TEMP_FILE" "$EASYBUILD_HOOKS" From b5ddeab77f5a8395c8096d178cd8920e54ad0b9c Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 12 Jun 2025 08:49:10 +0200 Subject: [PATCH 2/2] Update .github/workflows/test-eb-hooks.yml --- .github/workflows/test-eb-hooks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/test-eb-hooks.yml b/.github/workflows/test-eb-hooks.yml index aa54655a..e9aada40 100644 --- a/.github/workflows/test-eb-hooks.yml +++ b/.github/workflows/test-eb-hooks.yml @@ -51,8 +51,6 @@ jobs: fi # Compare the hooks to what is shipped in the repository - # (it is overkill, but harmless, to do this for every architecture) - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash module load EESSI-extend diff "$TEMP_FILE" "$EASYBUILD_HOOKS"