Make EESSI-extend support accelerator installations

.github/workflows/scripts/verify_eessi_environment.py

@@ -45,8 +45,8 @@ def check_env_endswith(var1, var2):
             check_env_equals("EESSI_ACCELERATOR_TARGET_OVERRIDE", "EESSI_ACCEL_SUBDIR")
             # special case is where EESSI_ACCELERATOR_TARGET_OVERRIDE may not match the final
             # accelerator architecture chosen.
-            # In CI we set FINAL_ACCELERATOR_PATH_EXPECTED to allow us to compare against an expected value.
-            check_env_equals("EESSI_ACCELERATOR_TARGET", "FINAL_ACCELERATOR_PATH_EXPECTED")
+            # In CI we set FINAL_ACCELERATOR_TARGET_EXPECTED to allow us to compare against an expected value.
+            check_env_equals("EESSI_ACCELERATOR_TARGET", "FINAL_ACCELERATOR_TARGET_EXPECTED")
         # verify the software paths that should exist
         check_env_endswith("EESSI_SOFTWARE_PATH", "EESSI_SOFTWARE_SUBDIR")
         check_env_endswith("EESSI_SITE_SOFTWARE_PATH", "EESSI_SOFTWARE_SUBDIR")

.github/workflows/tests_eessi_extend_module.yml

@@ -105,7 +105,8 @@ jobs:
           export EESSI_PROJECT_INSTALL="$MY_INSTALLATION_PATH"
           module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
           # check some specific envvars
-          check_env_var "EASYBUILD_INSTALLPATH" "$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
+          export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
+          check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"
           check_env_var "EASYBUILD_UMASK" "002"
           check_env_var "EASYBUILD_GROUP_WRITABLE_INSTALLDIR" "1"
           # unload and check the environment is clean again
@@ -118,10 +119,92 @@ jobs:
           mkdir -p $EESSI_USER_INSTALL  # must exist
           module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
           # check some specific envvars
-          check_env_var "EASYBUILD_INSTALLPATH" "$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
+          export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
+          check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"
           check_env_var "EASYBUILD_UMASK" "077"
           # unload and check the environment is clean again
           module unload EESSI-extend
           check_disallowed_env_prefix EASYBUILD_
           unset EESSI_USER_INSTALL
 
+      - name: Run tests for EESSI-extend in the various GPU scenarios
+        run: |
+          export MY_INSTALLATION_PATH=/tmp/easybuild
+
+          # Define a function to check the values of environment variables
+          # and another that checks an environment does not contain environment 
+          # variables matching a certain pattern
+          source .github/workflows/scripts/test_utils.sh
+
+          # Set an environment variable to use when we want to target accelerators
+          export STORED_EESSI_ACCELERATOR_TARGET_OVERRIDE="accel/nvidia/cc80"
+          export STORED_CUDA_CC="8.0"
+
+          # Let's start from a clean slate
+          module purge
+          export EESSI_ACCELERATOR_TARGET_OVERRIDE=$STORED_EESSI_ACCELERATOR_TARGET_OVERRIDE
+          module load EESSI/${{matrix.EESSI_VERSION}}
+          # Access the installed EESSI-extend
+          module use "$MY_INSTALLATION_PATH"/modules/all
+          check_disallowed_env_prefix EASYBUILD_
+
+          # Configure for CVMFS install
+          export EESSI_CVMFS_INSTALL=1
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
+          check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SOFTWARE_PATH"  # installation path should be the same unless we ask for an explicit GPU installation
+          check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC"
+          export EESSI_ACCELERATOR_INSTALL=1
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an actual GPU installation
+          check_env_var "EASYBUILD_INSTALLPATH" "${EESSI_SOFTWARE_PATH}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}"
+          # unload and make sure the environment is clean again
+          module unload EESSI-extend
+          check_disallowed_env_prefix EASYBUILD_
+          unset EESSI_ACCELERATOR_INSTALL
+          unset EESSI_CVMFS_INSTALL
+
+          # Now configure for a site
+          export EESSI_SITE_INSTALL=1
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
+          check_env_var "EASYBUILD_INSTALLPATH" "$EESSI_SITE_SOFTWARE_PATH"  # installation path should be the same unless we ask for an explicit GPU installation
+          check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC"
+          export EESSI_ACCELERATOR_INSTALL=1
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an actual GPU installation
+          check_env_var "EASYBUILD_INSTALLPATH" "${EESSI_SITE_SOFTWARE_PATH}/${EESSI_ACCELERATOR_TARGET_OVERRIDE}"
+          # unload and make sure the environment is clean again
+          module unload EESSI-extend
+          check_disallowed_env_prefix EASYBUILD_
+          unset EESSI_ACCELERATOR_INSTALL
+          unset EESSI_SITE_INSTALL
+
+          # Now for a project
+          export EESSI_PROJECT_INSTALL="$MY_INSTALLATION_PATH"
+          export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
+          check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"  # installation path should be the same unless we ask for an explicit GPU installation
+          check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC"
+          export EESSI_ACCELERATOR_INSTALL=1
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an GPU actual installation
+          check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"  # installation path should be the same for project case
+          # unload and make sure the environment is clean again
+          module unload EESSI-extend
+          check_disallowed_env_prefix EASYBUILD_
+          unset EESSI_ACCELERATOR_INSTALL
+          unset EESSI_PROJECT_INSTALL
+
+          # Now for a user
+          export EESSI_USER_INSTALL="$MY_INSTALLATION_PATH/$USER"
+          mkdir -p $EESSI_USER_INSTALL  # must exist
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
+          # check some specific envvars
+          export EXPECTED_INSTALLATION_PATH="$MY_INSTALLATION_PATH/$USER/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR"
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild
+          check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"  # installation path should be the same unless we ask for an explicit GPU installation
+          check_env_var "EASYBUILD_CUDA_COMPUTE_CAPABILITIES" "$STORED_CUDA_CC"
+          export EESSI_ACCELERATOR_INSTALL=1
+          module load EESSI-extend/${{matrix.EESSI_VERSION}}-easybuild # reload for an actual GPU installation
+          check_env_var "EASYBUILD_INSTALLPATH" "$EXPECTED_INSTALLATION_PATH"  # installation path should be the same for user case
+          # unload and make sure the environment is clean again
+          module unload EESSI-extend
+          check_disallowed_env_prefix EASYBUILD_
+          unset EESSI_ACCELERATOR_INSTALL
+          unset EESSI_USER_INSTALL     

.github/workflows/tests_eessi_module.yml

@@ -165,9 +165,9 @@ jobs:
         include:
           # For each override we expect a specific path (which may differ from the original due to overrides)
         - EESSI_ACCELERATOR_TARGET_OVERRIDE: accel/nvidia/cc80
-          FINAL_ACCELERATOR_PATH_EXPECTED: accel/nvidia/cc80
+          FINAL_ACCELERATOR_TARGET_EXPECTED: accel/nvidia/cc80
         - EESSI_ACCELERATOR_TARGET_OVERRIDE: accel/nvidia/cc77  # deliberately chose a non-existent CUDA capability
-          FINAL_ACCELERATOR_PATH_EXPECTED: accel/nvidia/cc70  # this reverts to the fallback case (which does exist)
+          FINAL_ACCELERATOR_TARGET_EXPECTED: accel/nvidia/cc70  # this reverts to the fallback case (which does exist)
 
     steps:
       - name: Check out software-layer repository
@@ -193,7 +193,7 @@ jobs:
           # Set our accelerator path overrides according to our matrix
           if [[ "${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}}" != "none" ]]; then
               export EESSI_ACCELERATOR_TARGET_OVERRIDE=${{matrix.EESSI_ACCELERATOR_TARGET_OVERRIDE}}
-              export FINAL_ACCELERATOR_PATH_EXPECTED=${{matrix.FINAL_ACCELERATOR_PATH_EXPECTED}}
+              export FINAL_ACCELERATOR_TARGET_EXPECTED=${{matrix.FINAL_ACCELERATOR_TARGET_EXPECTED}}
           fi
 
           # Turn on debug output in case we want to take a look

EESSI-extend-easybuild.eb

@@ -35,6 +35,11 @@ description = """
  If both EESSI_USER_INSTALL and EESSI_PROJECT_INSTALL are defined, both sets of
  installations are exposed, but new installations are created as user
  installations.
+
+ Strict installation path checking is enforced by EESSI for EESSI and site
+ installations involving accelerators. In these cases, if you wish to create an
+ accelerator installation you must set the environement variable
+ EESSI_ACCELERATOR_INSTALL (and load/reload this module).
 """
 
 toolchain = SYSTEM
@@ -78,8 +83,21 @@ if (mode() == "load") then
   end
 end
 working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER"))
+
 -- Gather the EPREFIX to use as a sysroot
 sysroot = os.getenv("EESSI_EPREFIX")
+
+-- Check if we have GPU capabilities and configure CUDA compute capabilities
+eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET")
+if (eessi_accelerator_target ~= nil) then
+  cuda_compute_capability = string.match(eessi_accelerator_target, "^accel/nvidia/cc([0-9][0-9])$")
+  if (cuda_compute_capability ~= nil) then
+    easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) ..  "." .. cuda_compute_capability:sub(2, 2)
+  else
+    LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target)
+  end
+end
+
 -- Use an installation prefix that we _should_ have write access to
 if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then
   -- Make sure no other EESSI install environment variables are set
@@ -88,22 +106,20 @@ if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then
   end
   eessi_cvmfs_install = true
   easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH")
-  eessi_accelerator_target = os.getenv("EESSI_ACCELERATOR_TARGET")
-  if (eessi_accelerator_target ~= nil) then
-    cuda_compute_capability = string.match(eessi_accelerator_target, "^nvidia/cc([0-9][0-9])$")
-    if (cuda_compute_capability ~= nil) then
-      easybuild_installpath = pathJoin(easybuild_installpath, 'accel', eessi_accelerator_target)
-      easybuild_cuda_compute_capabilities = cuda_compute_capability:sub(1, 1) ..  "." .. cuda_compute_capability:sub(2, 2)
-    else
-      LmodError("Incorrect value for $EESSI_ACCELERATOR_TARGET: " .. eessi_accelerator_target)
-    end
+  -- enforce accelerator subdirectory usage for CVMFS installs (only if an accelerator install is requested)
+  if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then
+      easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target)
   end
 elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then
   -- Make sure no other EESSI install environment variables are set
   if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then
     LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
   end
   easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH")
+  -- enforce accelerator subdirectory usage for site installs (only if an accelerator install is requested)
+  if (eessi_accelerator_target ~= nil) and (cuda_compute_capability ~= nil) and (os.getenv("EESSI_ACCELERATOR_INSTALL") ~= nil) then
+      easybuild_installpath = pathJoin(easybuild_installpath, eessi_accelerator_target)
+  end
 else
   -- Deal with user and project installs
   project_install = os.getenv("EESSI_PROJECT_INSTALL")

run_in_compat_layer_env.sh

@@ -29,8 +29,8 @@ fi
 if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}"
 fi
-if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then
-    INPUT="export EESSI_ACCELERATOR_TARGET=${EESSI_ACCELERATOR_TARGET}; ${INPUT}"
+if [ ! -z ${EESSI_ACCELERATOR_TARGET_OVERRIDE} ]; then
+    INPUT="export EESSI_ACCELERATOR_TARGET_OVERRIDE=${EESSI_ACCELERATOR_TARGET_OVERRIDE}; ${INPUT}"
 fi
 if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
     INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}"