Test if non CUDA builds are not added to accelorator path with jax

.github/workflows/parse_missing-installations-output.py

@@ -0,0 +1,23 @@
+import os
+import re
+
+missing = os.environ['missing']
+missing = missing.split('\n')
+missing_cuda = []
+missing_cpu = []
+for ec in missing:
+    if re.search('CUDA', ec):
+        missing_cuda.append(ec)
+    else:
+        missing_cpu.append(ec)
+if len(missing_cpu) != 0 and len(missing_cuda) != 0:
+    print(f'Please open a seperate pr for these dependencies: {missing_cpu}')
+    os.write(2, b'Error: CPU dependencies for CUDA build must be build in a seperate pr')
+    exit(1)
+elif len(missing_cuda) != 0:
+    # TODO: Make this set the accelorator label?
+    print(f'Have fun installing the following gpu builds: {missing_cuda}')
+elif len(missing_cpu) != 0:
+    print(f'Have fun installing the following gpu builds: {missing_cpu}')
+else:
+    print('no missing modules')

.github/workflows/scripts/filter_git_diff.py

@@ -0,0 +1,34 @@
+# At the moment this script will only filter out the easystack files
+# Next step could be to filter out other subpaths 
+# by using the GITHUB action envitonment variables and passing them to this python script.
+
+# Can also distiguish between modified, added files
+# That information is in the string that is generated by git diff
+# For filtering out changed easystacks this was however not required
+
+import os
+
+diff = os.getenv('CHANGED')
+env_file = os.getenv('GITHUB_ENV')
+
+diff_list = diff.split('\n')
+
+diff_filter_path = 'easystacks'
+
+diff_filtered = ''
+
+for line in diff_list:
+    status = line.split('\t')[0]
+    file = line.split('\t')[1] 
+    if file.startswith(diff_filter_path):
+        # Ignoring the status assigned to the file
+        diff_filtered += file + ' '
+
+if diff_filtered != '':
+    # If we will at some point add paths to in diff_filter_path
+    # we'll have to remove '/'
+    env_var = 'CHANGED_' + diff_filter_path.upper()
+    set_var = env_var + "=" + diff_filtered
+
+    with open(env_file, 'a') as file:
+        file.write(set_var)

.github/workflows/test-check_easystacks.yml

@@ -0,0 +1,108 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Check for CUDA builds in changed easystacks
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+env:
+  EESSI_ACCELERATOR_TARGETS: |
+    x86_64/amd/zen2:
+      - nvidia/cc80
+    x86_64/amd/zen3:
+      - nvidia/cc80
+jobs:
+  check_missing:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+        - 2023.06
+        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
+        - x86_64/amd/zen2
+        - x86_64/amd/zen3
+        - x86_64/amd/zen4
+        - x86_64/intel/haswell
+        - x86_64/intel/skylake_avx512
+        - x86_64/generic
+    steps:
+        - name: Check out software-layer repositor
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+            fetch-depth: 0
+        - name: Check for changed easystackfiles in pr
+          run: |
+            #Keep this for now to check once I started testing this with the upstream repo
+            git remote -v
+            echo "GITHUB_BASE_REF = ${GITHUB_BASE_REF}"
+            echo $GITHUB_ENV
+            echo "git diff --no-renames --name-status ${GITHUB_BASE_REF}..HEAD"
+            export CHANGED=$(git diff --no-renames --name-status origin/${GITHUB_BASE_REF}..HEAD)
+            echo $CHANGED
+            python .github/workflows/scripts/filter_git_diff.py
+        - name: Mount EESSI CernVM-FS pilot repository
+          if: env.CHANGED_EASYSTACKS
+          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+          with:
+              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+              cvmfs_http_proxy: DIRECT
+              cvmfs_repositories: software.eessi.io
+        - name: Test check_missing_installations.sh script
+          if: env.CHANGED_EASYSTACKS
+          run: |
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
+              # to prevent issues with checks in the Easybuild configuration that use this variable
+              export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
+              module load EasyBuild
+              which eb
+              eb --version
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_OS_TYPE=linux
+              env | grep ^EESSI | sort
+
+              # check for missing installations if a file in the easystacks directory is changed
+              echo "check for missing installation in changed easystack files"
+              easystack_files=(${CHANGED_EASYSTACKS})
+              echo $easystack_files
+
+              for easystack_file in ${easystack_files[@]}; do
+                  echo "check missing installatios for ${easystack_file}..."
+                  ./check_missing_installations.sh ${easystack_file} || ec=$?
+                  export eb_missing_out=$(./check_missing_installations.sh ${easystack_file})
+                  if [[ ${ec} -ne 0 ]]; then 
+                      echo "missing installations found for ${easystack_file}!" >&2
+                      export missing=$(env | grep .eb\) | tr -d \*)
+                      python .github/workflows/parse_missing-installations-output.py 
+                  fi
+              done
+
+              # now check the accelerator builds for this CPU target
+              #accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]")
+              #if [ -z ${accelerators} ]; then
+              #    echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+              #else
+              #    for accel in ${accelerators}; do
+              #        module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
+              #        echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
+              #        for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
+              #            echo "check missing installations for ${easystack_file}..."
+              #            ./check_missing_installations.sh ${easystack_file}
+              #            ec=$?
+              #            if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
+              #        done
+              #        module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
+              #    done
+              #fi */
+
+              #if ./check_missing_installations.sh ${easystack_file}; then
+              #  echo "did NOT capture missing package; test FAILED"
+              #  exit 1
+              #else
+              #  echo "captured missing package; test PASSED"
+                exit 0
+              fi

.github/workflows/test-software.eessi.io.yml

@@ -90,8 +90,8 @@ jobs:
                   module load EasyBuild/${eb_version}
                   which eb
                   eb --version
-                  ./check_missing_installations.sh ${easystack_file}
-                  ec=$?
+                  ec=0
+                  ./check_missing_installations.sh ${easystack_file} || ec=$?
                   if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
               done
 
@@ -110,8 +110,8 @@ jobs:
                           module load EasyBuild/${eb_version}
                           which eb
                           eb --version
-                          ./check_missing_installations.sh ${easystack_file}
-                          ec=$?
+                          ec=0
+                          ./check_missing_installations.sh ${easystack_file} || ec=$?
                           if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
                       done
                       module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all

.github/workflows/test_new_action_based_on_dorny.yml

@@ -0,0 +1,65 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Test new action
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+env:
+  EESSI_ACCELERATOR_TARGETS: |
+    x86_64/amd/zen2:
+      - nvidia/cc80
+    x86_64/amd/zen3:
+      - nvidia/cc80
+jobs:
+  check_missing:
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+        - 2023.06
+        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
+        - x86_64/amd/zen2
+        - x86_64/amd/zen3
+        - x86_64/amd/zen4
+        - x86_64/intel/haswell
+        - x86_64/intel/skylake_avx512
+        - x86_64/generic
+    steps:
+        - name: Check out software-layer repository
+          uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+            fetch-depth: 0
+        - name: find changed easystacks
+          run: |
+            #Keep this for now to check once I started testing this with the upstream repo
+            git remote -v
+            echo "GITHUB_BASE_REF = ${GITHUB_BASE_REF}"
+            echo $GITHUB_ENV
+            echo "git diff --no-renames --name-status ${GITHUB_BASE_REF}..HEAD"
+            export CHANGED=$(git diff --no-renames --name-status origin/${GITHUB_BASE_REF}..HEAD)
+            echo $CHANGED
+            python .github/workflows/scripts/filter_git_diff.py
+        - name: Are there changed files
+          if: env.CHANGED_EASYSTACKS
+          run: |
+            echo $CHANGED_EASYSTACKS
+        - name: Mount EESSI CernVM-FS pilot repository
+          if: env.CHANGED_EASYSTACKS
+          uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+          with:
+              cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+              cvmfs_http_proxy: DIRECT
+              cvmfs_repositories: software.eessi.io
+#        - name: find changed easystacks
+#          uses: laraPPr/test_easystack_filter_action@0c1cb269a89654084c59308e40ae92e90c854fba
+#          uses: dorny/paths-filter@v3
+#          id: changes
+#          with:
+#            list-files: shell
+#            filters: |
+#              changed:
+#                - 'easystacks/**'

easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml

@@ -5,3 +5,4 @@ easyconfigs:
       options:
         # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699
         from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3
+  - jax-0.4.25-gfbf-2023a-CUDA-12.1.1.eb