From adc31396d56f5573912a6e67186dfdc2cd9e1d79 Mon Sep 17 00:00:00 2001 From: vsc46128 vscuser Date: Thu, 27 Feb 2025 13:31:19 +0100 Subject: [PATCH 1/7] Add workflow to check if missing dependencies for a CUDA build include cpu builds --- .../parse_missing-installations-output.py | 23 ++++++ .github/workflows/test-check_easystacks.yml | 77 +++++++++++++++++++ .github/workflows/test-software.eessi.io.yml | 6 +- 3 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/parse_missing-installations-output.py create mode 100644 .github/workflows/test-check_easystacks.yml diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py new file mode 100644 index 0000000000..a5d42ca1e3 --- /dev/null +++ b/.github/workflows/parse_missing-installations-output.py @@ -0,0 +1,23 @@ +import os +import re + +missing = os.environ['missing'] +missing = missing.split('\n') +missing_cuda = [] +missing_cpu = [] +for ec in missing: + if re.search('CUDA', ec): + missing_cuda.append(ec) + else: + missing_cpu.append(ec) +if len(missing_cpu) != 0 and len(missing_cuda) != 0: + print(f'Please open a seperate pr for these dependencies: {missing_cpu}') + os.write(2, b'Error: CPU dependencies for CUDA build must be build in a seperate pr') + exit(1) +elif len(missing_cuda) != 0: + # TODO: Make this set the accelorator label? + print(f'Have fun installing the following gpu builds: {missing_cuda}') +elif len(missing_cpu) != 0: + print(f'Have fun installing the following gpu builds: {missing_cpu}') +else: + print('no missing modules') diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml new file mode 100644 index 0000000000..9fb5c80e6d --- /dev/null +++ b/.github/workflows/test-check_easystacks.yml @@ -0,0 +1,77 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check for CUDA builds in changed easystacks +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 +jobs: + check_missing: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + EESSI_SOFTWARE_SUBDIR_OVERRIDE: + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/generic + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 + with: + cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb + cvmfs_http_proxy: DIRECT + cvmfs_repositories: software.eessi.io + - name: find changed easystacks + uses: dorny/paths-filter@v3 + id: changes + with: + list-files: shell + filters: | + changed: + - 'easystacks/**' + - name: Test check_missing_installations.sh script + if: ${{ steps.changes.outputs.changed == 'true' }} + run: | + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} + source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash + # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash), + # to prevent issues with checks in the Easybuild configuration that use this variable + export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*} + module load EasyBuild + which eb + eb --version + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + env | grep ^EESSI | sort + + # check for missing installations if a file in the easystacks directory is changed + echo "check for missing installation in changed easystack files" + easystack_files=(${{ steps.changes.outputs.changed_files }}) + + for easystack_file in ${easystack_files[@]}; do + echo "check missing installatios for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} || ec=$? + export eb_missing_out=$(./check_missing_installations.sh ${easystack_file}) + if [[ ${ec} -ne 0 ]]; then + echo "missing installations found for ${easystack_file}!" >&2 + export missing=$(env | grep .eb\) | tr -d \*) + python .github/workflows/parse_missing-installations-output.py + fi + done diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index b8104a543f..020c4c31cc 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -60,8 +60,7 @@ jobs: echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." - ./check_missing_installations.sh ${easystack_file} - ec=$? + ./check_missing_installations.sh ${easystack_file} || ec=$? if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done @@ -75,8 +74,7 @@ jobs: echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." - ./check_missing_installations.sh ${easystack_file} - ec=$? + ./check_missing_installations.sh ${easystack_file} || ec=$? if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all From 3282614ab36e5f6deaa7cc9147172c5bdaac02b1 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 28 Feb 2025 12:27:42 +0100 Subject: [PATCH 2/7] add specific commit of dorny/paths-filter Co-authored-by: ocaisa --- .github/workflows/test-check_easystacks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 9fb5c80e6d..42c56ce6f9 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -39,7 +39,7 @@ jobs: cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - name: find changed easystacks - uses: dorny/paths-filter@v3 + uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2 id: changes with: list-files: shell From 1100c14e24e521b825e4cea5b3d0b33b8ad863eb Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Mon, 2 Jun 2025 11:36:22 +0200 Subject: [PATCH 3/7] use ec=0 Co-authored-by: ocaisa --- .github/workflows/test-software.eessi.io.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 020c4c31cc..1f08fa4180 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -60,6 +60,7 @@ jobs: echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." + ec=0 ./check_missing_installations.sh ${easystack_file} || ec=$? if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done From 7fb886e54103333d72efff1a079c52317ded9137 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Mon, 2 Jun 2025 11:36:51 +0200 Subject: [PATCH 4/7] =?UTF-8?q?use=20ec=3D=C2=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: ocaisa --- .github/workflows/test-software.eessi.io.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 1f08fa4180..d2a63d0b4c 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -75,6 +75,7 @@ jobs: echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do echo "check missing installations for ${easystack_file}..." + ec=0 ./check_missing_installations.sh ${easystack_file} || ec=$? if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done From e7b09f0402c89e3c2d2a096f98d1fdec803b8de5 Mon Sep 17 00:00:00 2001 From: laraPPr Date: Fri, 6 Jun 2025 17:16:18 +0200 Subject: [PATCH 5/7] add new filter of easystacks --- .github/workflows/scripts/filter_git_diff.py | 37 ++++++++++++++++++++ .github/workflows/test-check_easystacks.yml | 26 +++++++------- 2 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/scripts/filter_git_diff.py diff --git a/.github/workflows/scripts/filter_git_diff.py b/.github/workflows/scripts/filter_git_diff.py new file mode 100644 index 0000000000..8ea1ce4bcb --- /dev/null +++ b/.github/workflows/scripts/filter_git_diff.py @@ -0,0 +1,37 @@ +# At the moment this script will only filter out the easystack files +# Next step could be to filter out other subpaths +# by using the GITHUB action envitonment variables and argsparse. + +# Can also distiguish between modified, added, ... files +# That information is in the string that is generated by git diff +# For filtering easystacks this was however not necessary +# since we want to check both added and modified files + +import os + +diff = os.getenv('CHANGED') +env_file = os.getenv('GITHUB_ENV') + +diff_list = diff.split('\n') + +diff_filter_path = 'easystacks' + +diff_filtered = '' + +for line in diff_list: + status = line.split('\t')[0] + file = line.split('\t')[1] + if file.startswith(diff_filter_path): + # Ignoring the status assigned to the file + diff_filtered += file + ' ' + +if diff_filtered != '': + # If we will at some point add paths to in diff_filter_path + # we'll have to remove '/' + # Name of the env_var can than also change based on the file or path + env_var = 'CHANGED_' + diff_filter_path.upper() + set_var = env_var + "=" + diff_filtered + + # This adds the environment variable to the github action environment + with open(env_file, 'a') as file: + file.write(set_var) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index 42c56ce6f9..d51bb9fd83 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -1,5 +1,5 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Check for CUDA builds in changed easystacks +name: Check new and changed easystacks on: push: branches: [ "*-software.eessi.io" ] @@ -31,23 +31,25 @@ jobs: steps: - name: Check out software-layer repository uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - + with: + fetch-depth: 0 + - name: Check for changed easystackfiles in pr + run: | + echo "GITHUB_BASE_REF = ${GITHUB_BASE_REF}" + echo $GITHUB_ENV + echo "git diff --no-renames --name-status ${GITHUB_BASE_REF}..HEAD" + export CHANGED=$(git diff --no-renames --name-status origin/${GITHUB_BASE_REF}..HEAD) + echo $CHANGED + python .github/workflows/scripts/filter_git_diff.py - name: Mount EESSI CernVM-FS pilot repository + if: env.CHANGED_EASYSTACKS uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 with: cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb cvmfs_http_proxy: DIRECT cvmfs_repositories: software.eessi.io - - name: find changed easystacks - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 #v3.0.2 - id: changes - with: - list-files: shell - filters: | - changed: - - 'easystacks/**' - name: Test check_missing_installations.sh script - if: ${{ steps.changes.outputs.changed == 'true' }} + if: env.CHANGED_EASYSTACKS run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash @@ -63,7 +65,7 @@ jobs: # check for missing installations if a file in the easystacks directory is changed echo "check for missing installation in changed easystack files" - easystack_files=(${{ steps.changes.outputs.changed_files }}) + easystack_files=(${CHANGED_EASYSTACKS}) for easystack_file in ${easystack_files[@]}; do echo "check missing installatios for ${easystack_file}..." From 95259394b04a85d9128b1a943b811dcab8ec3916 Mon Sep 17 00:00:00 2001 From: laraPPr Date: Tue, 24 Jun 2025 17:05:03 +0200 Subject: [PATCH 6/7] remove the scripts who are also in software-layer-scripts and update the workflow Signed-off-by: laraPPr --- .../parse_missing-installations-output.py | 23 ------------ .github/workflows/scripts/filter_git_diff.py | 37 ------------------- .github/workflows/test-check_easystacks.yml | 4 +- 3 files changed, 2 insertions(+), 62 deletions(-) delete mode 100644 .github/workflows/parse_missing-installations-output.py delete mode 100644 .github/workflows/scripts/filter_git_diff.py diff --git a/.github/workflows/parse_missing-installations-output.py b/.github/workflows/parse_missing-installations-output.py deleted file mode 100644 index a5d42ca1e3..0000000000 --- a/.github/workflows/parse_missing-installations-output.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import re - -missing = os.environ['missing'] -missing = missing.split('\n') -missing_cuda = [] -missing_cpu = [] -for ec in missing: - if re.search('CUDA', ec): - missing_cuda.append(ec) - else: - missing_cpu.append(ec) -if len(missing_cpu) != 0 and len(missing_cuda) != 0: - print(f'Please open a seperate pr for these dependencies: {missing_cpu}') - os.write(2, b'Error: CPU dependencies for CUDA build must be build in a seperate pr') - exit(1) -elif len(missing_cuda) != 0: - # TODO: Make this set the accelorator label? - print(f'Have fun installing the following gpu builds: {missing_cuda}') -elif len(missing_cpu) != 0: - print(f'Have fun installing the following gpu builds: {missing_cpu}') -else: - print('no missing modules') diff --git a/.github/workflows/scripts/filter_git_diff.py b/.github/workflows/scripts/filter_git_diff.py deleted file mode 100644 index 8ea1ce4bcb..0000000000 --- a/.github/workflows/scripts/filter_git_diff.py +++ /dev/null @@ -1,37 +0,0 @@ -# At the moment this script will only filter out the easystack files -# Next step could be to filter out other subpaths -# by using the GITHUB action envitonment variables and argsparse. - -# Can also distiguish between modified, added, ... files -# That information is in the string that is generated by git diff -# For filtering easystacks this was however not necessary -# since we want to check both added and modified files - -import os - -diff = os.getenv('CHANGED') -env_file = os.getenv('GITHUB_ENV') - -diff_list = diff.split('\n') - -diff_filter_path = 'easystacks' - -diff_filtered = '' - -for line in diff_list: - status = line.split('\t')[0] - file = line.split('\t')[1] - if file.startswith(diff_filter_path): - # Ignoring the status assigned to the file - diff_filtered += file + ' ' - -if diff_filtered != '': - # If we will at some point add paths to in diff_filter_path - # we'll have to remove '/' - # Name of the env_var can than also change based on the file or path - env_var = 'CHANGED_' + diff_filter_path.upper() - set_var = env_var + "=" + diff_filtered - - # This adds the environment variable to the github action environment - with open(env_file, 'a') as file: - file.write(set_var) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index d51bb9fd83..be1cd0cba4 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -40,7 +40,7 @@ jobs: echo "git diff --no-renames --name-status ${GITHUB_BASE_REF}..HEAD" export CHANGED=$(git diff --no-renames --name-status origin/${GITHUB_BASE_REF}..HEAD) echo $CHANGED - python .github/workflows/scripts/filter_git_diff.py + python software-layer-scripts/.github/workflows/scripts/filter_git_diff.py - name: Mount EESSI CernVM-FS pilot repository if: env.CHANGED_EASYSTACKS uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0 @@ -74,6 +74,6 @@ jobs: if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2 export missing=$(env | grep .eb\) | tr -d \*) - python .github/workflows/parse_missing-installations-output.py + python software-layer-scripts/.github/workflows/scripts/parse_missing-installations-output.py fi done From 93dca8284b626ab3c4893865bf9796466a0672c7 Mon Sep 17 00:00:00 2001 From: laraPPr Date: Fri, 27 Jun 2025 12:38:37 +0200 Subject: [PATCH 7/7] add check for the use of from-pr in the easystack file Signed-off-by: laraPPr --- .github/workflows/test-check_easystacks.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-check_easystacks.yml b/.github/workflows/test-check_easystacks.yml index be1cd0cba4..48cc41f212 100644 --- a/.github/workflows/test-check_easystacks.yml +++ b/.github/workflows/test-check_easystacks.yml @@ -63,11 +63,19 @@ jobs: export EESSI_OS_TYPE=linux env | grep ^EESSI | sort - # check for missing installations if a file in the easystacks directory is changed + # check the changed easystack files echo "check for missing installation in changed easystack files" easystack_files=(${CHANGED_EASYSTACKS}) - + + # check for missing installations if a file in the easystacks directory is changed + # check easystack file for the use of `--from-pr` for easystack_file in ${easystack_files[@]}; do + echo "check for the use of from-pr in ${easystack_file}" + if grep -q "from-pr" ${easystack_file}; then + echo "ERROR: from-pr is found in ${easystack_file} please use from-commit" >&2 + exit 1 + fi + echo "check missing installatios for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} || ec=$? export eb_missing_out=$(./check_missing_installations.sh ${easystack_file})