From 8a6cd1329fc6f547f771d465e992d3341cf07584 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 1 Nov 2024 11:26:52 +0100 Subject: [PATCH 001/112] Add EESSI_DEV_PROJET to prefix --- init/minimal_eessi_env | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5e513c3c9f..0f9d3c0223 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,7 +8,14 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io +# and add a subdirectory to $EESSI_PREFIX +if [ ! -z ${EESSI_DEV_PROJECT} ]; then + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ +else + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ +fi + if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From 0e9402bb33f39dd520bf85c99a04a887a18b7904 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 1 Nov 2024 16:57:09 +0100 Subject: [PATCH 002/112] Print EESSI_CVMFS_REPO to debug --- EESSI-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 65c43d5ac5..8c88ef0ff0 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -225,6 +225,7 @@ fi # for now, this just reinstalls all scripts. Note the most elegant, but works # Only run install_scripts.sh if not dev.eessi.io for security +echo "EESSI_CVMFS_REPO: '${EESSI_CVMFS_REPO}'" if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} fi From 888b8740ce229f2f03110812781f238f6199b6e7 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 4 Nov 2024 11:27:20 +0100 Subject: [PATCH 003/112] Check if running in subshell is the cause of some variables being reset --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 8c88ef0ff0..1c6c3a0be8 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -131,13 +131,13 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" # Run in a subshell, so that minimal_eessi_env doesn't change the shell environment for the rest of this script - ( + # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set source $TOPDIR/init/minimal_eessi_env # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - ) + fi echo ">> Setting up environment..." From 1af43f180b6341291bbd4e1ea43b16a07f9773dc Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 4 Nov 2024 16:47:42 +0100 Subject: [PATCH 004/112] Revert to run minimal_eessi_env in subshell and add log line --- EESSI-install-software.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 1c6c3a0be8..f8245a71aa 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -131,13 +131,14 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" # Run in a subshell, so that minimal_eessi_env doesn't change the shell environment for the rest of this script - + ( # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set source $TOPDIR/init/minimal_eessi_env - + + echo ">> Creating directory for software subdirectory override: ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - + ) fi echo ">> Setting up environment..." From 25e8a5fb7f01180ffdf564727cd25a6d187a6c00 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 4 Nov 2024 16:48:08 +0100 Subject: [PATCH 005/112] Fix condition to build with subdir in prefix in dev.eessi.io --- init/minimal_eessi_env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 0f9d3c0223..ced2a6c694 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -10,7 +10,7 @@ source ${EESSI_INIT_DIR_PATH}/eessi_defaults # If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io # and add a subdirectory to $EESSI_PREFIX -if [ ! -z ${EESSI_DEV_PROJECT} ]; then +if [ -z ${EESSI_DEV_PROJECT} ]; then export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ else export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ From 3e5e2af65f1c47179cd0010e3430100bff248357 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 4 Nov 2024 17:31:40 +0100 Subject: [PATCH 006/112] Add some logs to see where EESSI_CVMFS_REPO is being ovewritten --- EESSI-install-software.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f8245a71aa..25f0383787 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -134,7 +134,7 @@ else ( # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set source $TOPDIR/init/minimal_eessi_env - + echo ">> Creating directory for software subdirectory override: ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} @@ -226,7 +226,7 @@ fi # for now, this just reinstalls all scripts. Note the most elegant, but works # Only run install_scripts.sh if not dev.eessi.io for security -echo "EESSI_CVMFS_REPO: '${EESSI_CVMFS_REPO}'" +echo "EESSI_CVMFS_REPO_post: '${EESSI_CVMFS_REPO}'" if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} fi @@ -266,6 +266,7 @@ fi echo ">> Configuring EasyBuild..." +echo "EESSI_CVMFS_REPO_pre: '${EESSI_CVMFS_REPO}'" # Make sure that we use the EESSI_CVMFS_INSTALL # Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded From 722ca22d52cdd3fe02fb6c5a559c32e2d0bfdb22 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 10:01:17 +0100 Subject: [PATCH 007/112] Add more log output for EESSI_DEV_PROJECT variable --- EESSI-install-software.sh | 4 ++-- init/minimal_eessi_env | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 25f0383787..9524ef7f77 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -226,7 +226,7 @@ fi # for now, this just reinstalls all scripts. Note the most elegant, but works # Only run install_scripts.sh if not dev.eessi.io for security -echo "EESSI_CVMFS_REPO_post: '${EESSI_CVMFS_REPO}'" +echo "EESSI_CVMFS_REPO_pre: '${EESSI_CVMFS_REPO}'" if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} fi @@ -266,7 +266,7 @@ fi echo ">> Configuring EasyBuild..." -echo "EESSI_CVMFS_REPO_pre: '${EESSI_CVMFS_REPO}'" +echo "EESSI_CVMFS_REPO_post: '${EESSI_CVMFS_REPO}'" # Make sure that we use the EESSI_CVMFS_INSTALL # Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index ced2a6c694..5bcc48c818 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -10,6 +10,7 @@ source ${EESSI_INIT_DIR_PATH}/eessi_defaults # If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io # and add a subdirectory to $EESSI_PREFIX +echo "EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}" if [ -z ${EESSI_DEV_PROJECT} ]; then export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ else From 1c3f20a1c9cf5f9ca3e943f3b14de9f5937a63d0 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 10:15:00 +0100 Subject: [PATCH 008/112] export EESSI_DEV_PROJECT to compat layer env --- run_in_compat_layer_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index b4299c7a0d..e9a49cbe3d 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -35,6 +35,9 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_DEV_PROJECT} ]; then + INPUT="export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}; ${INPUT}" +fi if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi From 7c9dee85842243076d6376026571d7f6c7d978e8 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 10:54:38 +0100 Subject: [PATCH 009/112] Expose cvmfs and version override in eessi_defaults --- init/eessi_defaults | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/init/eessi_defaults b/init/eessi_defaults index 654a829425..bdc8cc4871 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -19,8 +19,12 @@ if [[ $(uname -m) == "riscv64" ]]; then echo "" fi else + echo "eessi_defaults cvmfs override ${EESSI_CVMFS_REPO_OVERRIDE}" + echo "eessi_defaults version override ${EESSI_VERSION_OVERRIDE}" export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" + echo "eessi_defaults cvmfs override post ${EESSI_CVMFS_REPO_OVERRIDE}" + echo "eessi_defaults version override post${EESSI_VERSION_OVERRIDE}" fi # use archdetect by default, unless otherwise specified export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}" From d5d152223e2a1b6fe221cdd1ea04f1f26579d427 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 10:55:56 +0100 Subject: [PATCH 010/112] Also expose EESSI_CVMFS_REPO in defaults --- init/eessi_defaults | 1 + 1 file changed, 1 insertion(+) diff --git a/init/eessi_defaults b/init/eessi_defaults index bdc8cc4871..fcb0f42317 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -25,6 +25,7 @@ else export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" echo "eessi_defaults cvmfs override post ${EESSI_CVMFS_REPO_OVERRIDE}" echo "eessi_defaults version override post${EESSI_VERSION_OVERRIDE}" + echo "eessi_defaults cvmfs repo post ${EESSI_CVMFS_REPO}" fi # use archdetect by default, unless otherwise specified export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}" From fb45d7e8cc1b7233c31953d6cc182b9a26994f9d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 12:02:09 +0100 Subject: [PATCH 011/112] Try exporting EESSI_CVMFS_INSTALL --- init/minimal_eessi_env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5bcc48c818..9861d1e3c9 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -15,6 +15,8 @@ if [ -z ${EESSI_DEV_PROJECT} ]; then export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ else export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ + # Set EESSI_PREFIX to the dev project path for EESSI-extend + export EESSI_CVMFS_INSTALL=$EESSI_PREFIX fi From a1f11feab78fbc10a556937695d9970ca68066e9 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 12:43:50 +0100 Subject: [PATCH 012/112] Overwrite EASYBUILD_INSTALLPATH and use EESSI_CVMFS_REPO_OVERRIDE in minimal_eessi_env --- EESSI-install-software.sh | 7 +++++-- init/minimal_eessi_env | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9524ef7f77..b93315336a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -207,6 +207,7 @@ export PYTHONPYCACHEPREFIX=$TMPDIR/pycache # - .lmod/lmodrc.lua # - .lmod/SitePackage.lua _eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +echo "_eessi_software_path: ${_eessi_software_path}" _lmod_cfg_dir=${_eessi_software_path}/.lmod _lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua if [ ! -f ${_lmod_rc_file} ]; then @@ -297,10 +298,12 @@ fi # If in dev.eessi.io, allow building on top of software.eessi.io if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all + # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io + EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ +else + module use $EASYBUILD_INSTALLPATH/modules/all fi -module use $EASYBUILD_INSTALLPATH/modules/all - if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 9861d1e3c9..abfda6d15a 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -14,7 +14,7 @@ echo "EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}" if [ -z ${EESSI_DEV_PROJECT} ]; then export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ else - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ + export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ # Set EESSI_PREFIX to the dev project path for EESSI-extend export EESSI_CVMFS_INSTALL=$EESSI_PREFIX fi From dd9fccd78f318bbfbed57bb65fd19bcaf896491b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 12:54:46 +0100 Subject: [PATCH 013/112] Load $EASYBUILD_INSTALLPATH modules in dev.eessi.io --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index b93315336a..2ee28c966a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -300,7 +300,7 @@ if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ -else + echo "\$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" module use $EASYBUILD_INSTALLPATH/modules/all fi From 821455bc8f25c0b0fcab8c40b89c7cd83ef29ee9 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 5 Nov 2024 14:09:57 +0100 Subject: [PATCH 014/112] Use EESSI_DEV_PROJECT for check instead of EESSI_CVMFS_REPO in EASYBUILD_INSTALLPATH override --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2ee28c966a..326be7061c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -296,7 +296,7 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi # If in dev.eessi.io, allow building on top of software.eessi.io -if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ From a19c79cb91fac5a567218580c677434c22f0b8a4 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 6 Nov 2024 17:10:29 +0100 Subject: [PATCH 015/112] Fix running install_scripts.sh only if not in dev.eessi.io --- EESSI-install-software.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 326be7061c..e85a8fb7bd 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -226,9 +226,8 @@ fi # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works -# Only run install_scripts.sh if not dev.eessi.io for security -echo "EESSI_CVMFS_REPO_pre: '${EESSI_CVMFS_REPO}'" -if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then +# Only run install_scripts.sh if not in dev.eessi.io for security +if [[ -z ${EESSI_DEV_PROJECT} ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} fi @@ -267,7 +266,6 @@ fi echo ">> Configuring EasyBuild..." -echo "EESSI_CVMFS_REPO_post: '${EESSI_CVMFS_REPO}'" # Make sure that we use the EESSI_CVMFS_INSTALL # Since the path is set when loading EESSI-extend, we reload it to make sure it works - even if it is already loaded @@ -300,7 +298,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ - echo "\$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" + echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" module use $EASYBUILD_INSTALLPATH/modules/all fi From afaa94fe9eb21497f2d2b18716b3122995d9867d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 6 Nov 2024 17:10:47 +0100 Subject: [PATCH 016/112] Remove debug printing to log --- init/eessi_defaults | 5 ----- 1 file changed, 5 deletions(-) diff --git a/init/eessi_defaults b/init/eessi_defaults index fcb0f42317..654a829425 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -19,13 +19,8 @@ if [[ $(uname -m) == "riscv64" ]]; then echo "" fi else - echo "eessi_defaults cvmfs override ${EESSI_CVMFS_REPO_OVERRIDE}" - echo "eessi_defaults version override ${EESSI_VERSION_OVERRIDE}" export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/software.eessi.io}" export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}" - echo "eessi_defaults cvmfs override post ${EESSI_CVMFS_REPO_OVERRIDE}" - echo "eessi_defaults version override post${EESSI_VERSION_OVERRIDE}" - echo "eessi_defaults cvmfs repo post ${EESSI_CVMFS_REPO}" fi # use archdetect by default, unless otherwise specified export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}" From 2c3e74e61ea38aa1c5624e5444b3f2508aec4b7f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 6 Nov 2024 17:15:44 +0100 Subject: [PATCH 017/112] Improved logging when $EESSI_DEV_PROJECT is set --- init/minimal_eessi_env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index abfda6d15a..6b57fc0079 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -10,10 +10,10 @@ source ${EESSI_INIT_DIR_PATH}/eessi_defaults # If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io # and add a subdirectory to $EESSI_PREFIX -echo "EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}" if [ -z ${EESSI_DEV_PROJECT} ]; then export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ else + echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ # Set EESSI_PREFIX to the dev project path for EESSI-extend export EESSI_CVMFS_INSTALL=$EESSI_PREFIX From 3d9424d3ac8b2e88ed41714a2cec013006e35319 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 8 Nov 2024 13:34:26 +0100 Subject: [PATCH 018/112] Fix tarball creations in dev.eessi.io --- create_tarball.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 01f498e1ac..fc0674d9bc 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -20,6 +20,16 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} +eessi_dev_project=${EESSI_DEV_PROJECT} + +# install to $eessi_version/$eessi_dev_project if defined, otherwise install +# to $eessi_version +echo "Setting $install_prefix_dir to: ${install_prefix_dir}" + +if [ ! -d ${install_prefix_dir} ]; then + echo "Install prefix directory ${install_prefix_dir} does not exist?!" >&2 + exit 2 +fi software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then @@ -62,18 +72,18 @@ fi # consider both CPU-only and accelerator subdirectories for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do - if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then + if [ -d ${install_prefix_dir}/software/${os}/${subdir}/modules ]; then # module files - find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match + find ${install_prefix_dir}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module symlinks - find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match + find ${install_prefix_dir}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module files and symlinks - find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ + find ${install_prefix_dir}/software/${os}/${subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi - if [ -d ${eessi_version}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then + if [ -d ${install_prefix_dir}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' # using EasyBuild's standard module naming scheme) match the name of the From 028f935f32c9bc7eb3a84f987c50159beaceab0d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 8 Nov 2024 14:16:24 +0100 Subject: [PATCH 019/112] Actually set install_prefix_dir --- create_tarball.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index fc0674d9bc..ee70baaca1 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -4,15 +4,16 @@ set -e base_dir=$(dirname $(realpath $0)) -if [ $# -ne 5 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ $# -ne 6 ]; then + echo "ERROR: Usage: $0 (example: ESPREsSo)" >&2 exit 1 fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 accel_subdir=$4 -target_tgz=$5 +eessi_dev_project=$5 +target_tgz=$6 tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -20,11 +21,11 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} -eessi_dev_project=${EESSI_DEV_PROJECT} # install to $eessi_version/$eessi_dev_project if defined, otherwise install # to $eessi_version -echo "Setting $install_prefix_dir to: ${install_prefix_dir}" +install_prefix_dir=${eessi_version}${eessi_dev_project:+/$eessi_dev_project}/ +echo "Setting Install prefix directory to: ${install_prefix_dir}" if [ ! -d ${install_prefix_dir} ]; then echo "Install prefix directory ${install_prefix_dir} does not exist?!" >&2 From 4f0686406092130d2f0f4cb1c6585808bfe3de28 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 8 Nov 2024 14:58:49 +0100 Subject: [PATCH 020/112] Log EESSI_DEV_PROJECT and export it --- bot/build.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 3fd343e96f..6f08d14013 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -283,6 +283,10 @@ timestamp=$(date +%s) source $software_layer_dir/init/eessi_defaults export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball +echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'" +export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} + # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh # TODO should we make this a configurable parameter of eessi_container.sh using @@ -290,8 +294,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" \"${EESSI_DEV_PROJECT}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" "${EESSI_DEV_PROJECT}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From 9eca5e4ad6d0f329235b8a3dfa50b58fc4a42ec6 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 8 Nov 2024 16:19:14 +0100 Subject: [PATCH 021/112] Fix test checking for ${install_prefix_dir} --- create_tarball.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index ee70baaca1..a9861b465f 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -27,8 +27,8 @@ cvmfs_repo=${EESSI_CVMFS_REPO} install_prefix_dir=${eessi_version}${eessi_dev_project:+/$eessi_dev_project}/ echo "Setting Install prefix directory to: ${install_prefix_dir}" -if [ ! -d ${install_prefix_dir} ]; then - echo "Install prefix directory ${install_prefix_dir} does not exist?!" >&2 +if [ -z ${install_prefix_dir} ]; then + echo "\${install_prefix_dir} is empty?!" >&2 exit 2 fi From 0e68f1823c0caec3de33ec3f87928acf057559f3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 10 Nov 2024 19:44:09 +0100 Subject: [PATCH 022/112] Look for installed software in the right directory --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index a9861b465f..1f6f40d0c6 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -93,7 +93,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" - ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ + ls -d ${install_prefix_dir}/software/${os}/${subdir}/software/${package_version} \ | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match done fi From 82d36c756a6e92271b3e068252334a1f561ad46f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 19 Nov 2024 10:23:06 +0100 Subject: [PATCH 023/112] Fix error reporting of input in create_tarball.sh --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 1f6f40d0c6..eea92305ed 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -5,7 +5,7 @@ set -e base_dir=$(dirname $(realpath $0)) if [ $# -ne 6 ]; then - echo "ERROR: Usage: $0 (example: ESPREsSo)" >&2 + echo "ERROR: Usage: $0 (example: ESPREsSo) " >&2 exit 1 fi eessi_tmpdir=$1 From 3248438f4f3969b499bcfdb193009fb25ba9cd9a Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 19 Nov 2024 10:46:57 +0100 Subject: [PATCH 024/112] Fix installpath to include cpu target --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e85a8fb7bd..fdfd1aded4 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -297,7 +297,7 @@ fi if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io - EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ + EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" module use $EASYBUILD_INSTALLPATH/modules/all fi From 4db10830c0365008ea29260b3829c109eb9c40bb Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 29 Nov 2024 14:49:14 +0100 Subject: [PATCH 025/112] Track EASYBUILD_EXPERIMENTAL set --- EESSI-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7484c5b479..17fe2933d6 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -253,6 +253,8 @@ export EESSI_CVMFS_INSTALL=1 # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} source load_eessi_extend_module.sh ${EESSI_VERSION} +echo ">> Loaded load_eessi_extend_module.sh ${EESSI_VERSION}" +echo ">> \$EASYBUILD_EXPERIMENTAL: ${EASYBUILD_EXPERIMENTAL}" # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works From 720d6e95481667672697832ea33154abbcda4355 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 29 Nov 2024 14:49:23 +0100 Subject: [PATCH 026/112] Fix typo in path --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index eea92305ed..0966a09b3e 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -24,7 +24,7 @@ cvmfs_repo=${EESSI_CVMFS_REPO} # install to $eessi_version/$eessi_dev_project if defined, otherwise install # to $eessi_version -install_prefix_dir=${eessi_version}${eessi_dev_project:+/$eessi_dev_project}/ +install_prefix_dir=${eessi_version}${eessi_dev_project:+/$eessi_dev_project} echo "Setting Install prefix directory to: ${install_prefix_dir}" if [ -z ${install_prefix_dir} ]; then From e126f78075c1b621da431b7e7e13688be24f3390 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 29 Nov 2024 15:06:34 +0100 Subject: [PATCH 027/112] Use $TOPDIR to load_eessi_extend_module.sh --- EESSI-install-software.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 17fe2933d6..88e999c68b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -252,9 +252,7 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -source load_eessi_extend_module.sh ${EESSI_VERSION} -echo ">> Loaded load_eessi_extend_module.sh ${EESSI_VERSION}" -echo ">> \$EASYBUILD_EXPERIMENTAL: ${EASYBUILD_EXPERIMENTAL}" +source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works From 5a7012caf071d271210e3cf11f221cb297d043c6 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 29 Nov 2024 15:51:06 +0100 Subject: [PATCH 028/112] Handle custom prefix in dev.eessi.io --- bot/check-build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index 41aeab258e..faa2ff1e05 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -480,6 +480,12 @@ if [[ ! -z ${TARBALL} ]]; then accelerator=$(cfg_get_value "architecture" "accelerator") prefix="${repo_version}/software/${os_type}/${software_subdir}" + # if building for dev.eessi.io is set, the prefix is different + # it needs the project name in the path + if [ ! -z ${EESSI_DEV_PROJECT} ]; then + prefix="${repo_version}/${EESSI_DEV_PROJECT}/software/${os_type}/${software_subdir}" + fi + # if we build for an accelerator, the prefix is different if [[ ! -z ${accelerator} ]]; then prefix="${prefix}/accel/${accelerator}" From 8dd50b098c3cd39f0558d61d87fa7608e5fd050e Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 29 Nov 2024 21:49:20 +0100 Subject: [PATCH 029/112] create_tarball handles software_dir with eessi_dev_project --- create_tarball.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0966a09b3e..0b6819e81a 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -32,7 +32,8 @@ if [ -z ${install_prefix_dir} ]; then exit 2 fi -software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" +# software_dir includes $eessi_dev_project if defined, otherwise install omit +software_dir="${cvmfs_repo}/versions/${eessi_version}${eessi_dev_project:+/$eessi_dev_project}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 From 1bec91730d08145dd8fccd7a3e95cb3882e9bb9c Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Fri, 17 Jan 2025 17:00:59 +0100 Subject: [PATCH 030/112] Fix merge conflict --- EESSI-install-software.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 88e999c68b..922303a390 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -135,9 +135,9 @@ else # Make sure EESSI_PREFIX and EESSI_OS_TYPE are set source $TOPDIR/init/minimal_eessi_env - echo ">> Creating directory for software subdirectory override: ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" - # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) - mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # make sure the the software and modules directory exist + # (since it's expected by init/eessi_environment_variables when using archdetect and by the EESSI module) + mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/{modules,software} ) fi From 5b477370302d861a3678324e5038ea1b7876411b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 27 Jan 2025 17:36:13 +0100 Subject: [PATCH 031/112] Remove trailing `/` --- init/minimal_eessi_env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 6b57fc0079..5150d49259 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -11,10 +11,10 @@ source ${EESSI_INIT_DIR_PATH}/eessi_defaults # If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io # and add a subdirectory to $EESSI_PREFIX if [ -z ${EESSI_DEV_PROJECT} ]; then - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION/ + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION else echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" - export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/ + export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT # Set EESSI_PREFIX to the dev project path for EESSI-extend export EESSI_CVMFS_INSTALL=$EESSI_PREFIX fi From 1ecd93b0433539de137047d777ddff3e61c0fd0a Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 31 Jan 2025 17:04:21 +0100 Subject: [PATCH 032/112] Set EESSI_DEV_PROJECT via EESSI-extend's EESSI_PROJECT_INSTALL --- EESSI-install-software.sh | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0b45ad018d..2d8152ece9 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -252,6 +252,11 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +# If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to +# /cvmfs/dev.eessi.io/ +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + $EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections @@ -293,15 +298,6 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi -# If in dev.eessi.io, allow building on top of software.eessi.io -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all - # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io - EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" - module use $EASYBUILD_INSTALLPATH/modules/all -fi - if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else From a4d61d6205592580ae0a27cd9dbbeb7c7fd406ad Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 31 Jan 2025 17:28:07 +0100 Subject: [PATCH 033/112] Add some debugging infop --- EESSI-install-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2d8152ece9..289df57eb3 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,7 +255,10 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then +echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" $EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 35bd8a18daae52d62910f59614f3f2751add5cf1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 31 Jan 2025 17:54:07 +0100 Subject: [PATCH 034/112] Fix syntax error --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 289df57eb3..67b09f2bb1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -256,10 +256,10 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" - $EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" - fi + source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections From 72ba311687fc30dbefcd1d60d749eb11792c4290 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 31 Jan 2025 18:03:18 +0100 Subject: [PATCH 035/112] Unset EESSI_CVMFS_INSTALL on dev.eessi.io builds --- EESSI-install-software.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 67b09f2bb1..854677cfbb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,7 +255,9 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then -echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" + echo "Unsetting EESSI_CVMFS_INSTALL" + unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From b3399f2464a90e675b41cd00dbab3a770c51afff Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sat, 1 Feb 2025 16:19:00 +0100 Subject: [PATCH 036/112] Shorten EESSI_PROJECT_INSTALL path --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 854677cfbb..9f10f48c10 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 9f0b2dffd101dcadc8e5f39651e68ce958f0b7ed Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sat, 1 Feb 2025 16:40:58 +0100 Subject: [PATCH 037/112] Remove more redundant parts of path --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9f10f48c10..e6869959e1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/$EESSI_DEV_PROJECT echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 3fc95e1ee3ce49f19d63fa3fbfad1064bc38866e Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sat, 1 Feb 2025 17:45:47 +0100 Subject: [PATCH 038/112] Restore right part of path --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e6869959e1..c165679b2c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/$EESSI_DEV_PROJECT + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 28c6b7ee0a85471064aac2ca3e9313975c6d5de1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sat, 1 Feb 2025 18:54:24 +0100 Subject: [PATCH 039/112] Use EESSI_SITE_INSTALL instead --- EESSI-install-software.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c165679b2c..102e95699a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION - echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 39d67ebca0c3f7c9d71fd73fa4a5530ea3c1d432 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 17:07:05 +0100 Subject: [PATCH 040/112] Try EESSI_PROJECT_INSTALL without minimal_eessi_env override --- EESSI-install-software.sh | 6 +++--- init/minimal_eessi_env | 12 +----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 102e95699a..9f10f48c10 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5150d49259..5e513c3c9f 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,17 +8,7 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io -# and add a subdirectory to $EESSI_PREFIX -if [ -z ${EESSI_DEV_PROJECT} ]; then - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION -else - echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" - export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - # Set EESSI_PREFIX to the dev project path for EESSI-extend - export EESSI_CVMFS_INSTALL=$EESSI_PREFIX -fi - +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From a37ae0b34aa8cf7a1cb185d83eecdc8d8dd93d98 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 18:59:35 +0100 Subject: [PATCH 041/112] Set EESSI_PROJECT_INSTALL in minimal_eessi_env --- init/minimal_eessi_env | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5e513c3c9f..954243cd57 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,7 +8,17 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io +# and add a subdirectory to $EESSI_PREFIX +if [ -z ${EESSI_DEV_PROJECT} ]; then + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +else + echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" + export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + # Set EESSI_PREFIX to the dev project path for EESSI-extend + export EESSI_PROJECT_INSTALL=$EESSI_PREFIX +fi + if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From 2145b935cba62ba2a5291799a09106814f1ad65e Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 19:28:18 +0100 Subject: [PATCH 042/112] Try full path --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9f10f48c10..854677cfbb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 145427fa6f5e46b52292608053e16f3703388b0b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 19:42:32 +0100 Subject: [PATCH 043/112] Revert "Try full path" This reverts commit 2145b935cba62ba2a5291799a09106814f1ad65e. --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 854677cfbb..9f10f48c10 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 751aa891af85d00bd82d1ecd06b9fdbbe3aff63f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 19:42:45 +0100 Subject: [PATCH 044/112] Revert "Set EESSI_PROJECT_INSTALL in minimal_eessi_env" This reverts commit a37ae0b34aa8cf7a1cb185d83eecdc8d8dd93d98. --- init/minimal_eessi_env | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 954243cd57..5e513c3c9f 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,17 +8,7 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io -# and add a subdirectory to $EESSI_PREFIX -if [ -z ${EESSI_DEV_PROJECT} ]; then - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION -else - echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" - export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - # Set EESSI_PREFIX to the dev project path for EESSI-extend - export EESSI_PROJECT_INSTALL=$EESSI_PREFIX -fi - +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From 5759cd521f54876821d9648c76e4156db809d800 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 19:42:48 +0100 Subject: [PATCH 045/112] Revert "Try EESSI_PROJECT_INSTALL without minimal_eessi_env override" This reverts commit 39d67ebca0c3f7c9d71fd73fa4a5530ea3c1d432. --- EESSI-install-software.sh | 6 +++--- init/minimal_eessi_env | 12 +++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9f10f48c10..102e95699a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5e513c3c9f..5150d49259 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,7 +8,17 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io +# and add a subdirectory to $EESSI_PREFIX +if [ -z ${EESSI_DEV_PROJECT} ]; then + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +else + echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" + export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + # Set EESSI_PREFIX to the dev project path for EESSI-extend + export EESSI_CVMFS_INSTALL=$EESSI_PREFIX +fi + if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From cc240d13913960f507c2f92d351663da4ad0b4ab Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 19:42:49 +0100 Subject: [PATCH 046/112] Revert "Use EESSI_SITE_INSTALL instead" This reverts commit 28c6b7ee0a85471064aac2ca3e9313975c6d5de1. --- EESSI-install-software.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 102e95699a..c165679b2c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION + echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 2769b6cd113ecf3468341601e4dc7f9cda76dcc3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:17 +0100 Subject: [PATCH 047/112] Reapply "Use EESSI_SITE_INSTALL instead" This reverts commit cc240d13913960f507c2f92d351663da4ad0b4ab. --- EESSI-install-software.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c165679b2c..102e95699a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION - echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 1fead90cd4979fceb36c90ab842b0e1cd9e1cc0f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:19 +0100 Subject: [PATCH 048/112] Reapply "Try EESSI_PROJECT_INSTALL without minimal_eessi_env override" This reverts commit 5759cd521f54876821d9648c76e4156db809d800. --- EESSI-install-software.sh | 6 +++--- init/minimal_eessi_env | 12 +----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 102e95699a..9f10f48c10 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5150d49259..5e513c3c9f 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,17 +8,7 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io -# and add a subdirectory to $EESSI_PREFIX -if [ -z ${EESSI_DEV_PROJECT} ]; then - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION -else - echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" - export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - # Set EESSI_PREFIX to the dev project path for EESSI-extend - export EESSI_CVMFS_INSTALL=$EESSI_PREFIX -fi - +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From 13b2190c5c43c18bf00cdf2675b0f3b17a894ea8 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:20 +0100 Subject: [PATCH 049/112] Reapply "Set EESSI_PROJECT_INSTALL in minimal_eessi_env" This reverts commit 751aa891af85d00bd82d1ecd06b9fdbbe3aff63f. --- init/minimal_eessi_env | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5e513c3c9f..954243cd57 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,7 +8,17 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io +# and add a subdirectory to $EESSI_PREFIX +if [ -z ${EESSI_DEV_PROJECT} ]; then + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +else + echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" + export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + # Set EESSI_PREFIX to the dev project path for EESSI-extend + export EESSI_PROJECT_INSTALL=$EESSI_PREFIX +fi + if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From 8b4b02232d74204f287c85b7c14b8862e4f14b7b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:20 +0100 Subject: [PATCH 050/112] Reapply "Try full path" This reverts commit 145427fa6f5e46b52292608053e16f3703388b0b. --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9f10f48c10..854677cfbb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 33a9d64bac41010b6703d288e397fa30e5c373c5 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:20 +0100 Subject: [PATCH 051/112] Revert "Try full path" This reverts commit 2145b935cba62ba2a5291799a09106814f1ad65e. --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 854677cfbb..9f10f48c10 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 228fe3b94c45ef8ad4e7f7b021d8a433f076f83f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:20 +0100 Subject: [PATCH 052/112] Revert "Set EESSI_PROJECT_INSTALL in minimal_eessi_env" This reverts commit a37ae0b34aa8cf7a1cb185d83eecdc8d8dd93d98. --- init/minimal_eessi_env | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 954243cd57..5e513c3c9f 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,17 +8,7 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io -# and add a subdirectory to $EESSI_PREFIX -if [ -z ${EESSI_DEV_PROJECT} ]; then - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION -else - echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" - export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - # Set EESSI_PREFIX to the dev project path for EESSI-extend - export EESSI_PROJECT_INSTALL=$EESSI_PREFIX -fi - +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From 72395ea8c421da2265ea7b460e54f852b45c37ff Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:20 +0100 Subject: [PATCH 053/112] Revert "Try EESSI_PROJECT_INSTALL without minimal_eessi_env override" This reverts commit 39d67ebca0c3f7c9d71fd73fa4a5530ea3c1d432. --- EESSI-install-software.sh | 6 +++--- init/minimal_eessi_env | 12 +++++++++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9f10f48c10..102e95699a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5e513c3c9f..5150d49259 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,7 +8,17 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io +# and add a subdirectory to $EESSI_PREFIX +if [ -z ${EESSI_DEV_PROJECT} ]; then + export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION +else + echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" + export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + # Set EESSI_PREFIX to the dev project path for EESSI-extend + export EESSI_CVMFS_INSTALL=$EESSI_PREFIX +fi + if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' From b5b9916da4890022b553e0d5412a6c4ff53e87ff Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:20 +0100 Subject: [PATCH 054/112] Revert "Use EESSI_SITE_INSTALL instead" This reverts commit 28c6b7ee0a85471064aac2ca3e9313975c6d5de1. --- EESSI-install-software.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 102e95699a..c165679b2c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,11 +255,11 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_SITE_INSTALL" + echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_SITE_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - echo "EESSI_SITE_INSTALL set to ${EESSI_SITE_INSTALL}" + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION + echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 00fc0258ae45f5214b2d12f0440d04c477c1ade3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 055/112] Revert "Restore right part of path" This reverts commit 3fc95e1ee3ce49f19d63fa3fbfad1064bc38866e. --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c165679b2c..e6869959e1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/$EESSI_DEV_PROJECT echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From cb41a856f65d38f1cbb6f5b87e22edc6a83b29e3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 056/112] Revert "Remove more redundant parts of path" This reverts commit 9f0b2dffd101dcadc8e5f39651e68ce958f0b7ed. --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index e6869959e1..9f10f48c10 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/$EESSI_DEV_PROJECT + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From ab7245adefe98a767894f2f8efdfdbc847182250 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 057/112] Revert "Shorten EESSI_PROJECT_INSTALL path" This reverts commit b3399f2464a90e675b41cd00dbab3a770c51afff. --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9f10f48c10..854677cfbb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,7 +258,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" echo "Unsetting EESSI_CVMFS_INSTALL" unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From d9bed4fbc8f8a70e9780cc359ac4b118d1098f3f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 058/112] Revert "Unset EESSI_CVMFS_INSTALL on dev.eessi.io builds" This reverts commit 72ba311687fc30dbefcd1d60d749eb11792c4290. --- EESSI-install-software.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 854677cfbb..67b09f2bb1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,9 +255,7 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" - echo "Unsetting EESSI_CVMFS_INSTALL" - unset EESSI_CVMFS_INSTALL +echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 2dd7650e10c2d61706a802b1577abf9e6373b173 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 059/112] Revert "Fix syntax error" This reverts commit 35bd8a18daae52d62910f59614f3f2751add5cf1. --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 67b09f2bb1..289df57eb3 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -256,10 +256,10 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + $EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" -fi +fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections From 5bb1c560472d40846c7035ceec76d02905366251 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 060/112] Revert "Add some debugging infop" This reverts commit a4d61d6205592580ae0a27cd9dbbeb7c7fd406ad. --- EESSI-install-software.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 289df57eb3..2d8152ece9 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,10 +255,7 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to # /cvmfs/dev.eessi.io/ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then -echo "EESSI_DEV_PROJECT is set to ${EESSI_DEV_PROJECT}, setting EESSI_PROJECT_INSTALL" $EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - echo "EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" - fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From dc95dbda0442f14886c0583dc1f8991b2eb8a649 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 2 Feb 2025 20:38:21 +0100 Subject: [PATCH 061/112] Revert "Set EESSI_DEV_PROJECT via EESSI-extend's EESSI_PROJECT_INSTALL" This reverts commit 1ecd93b0433539de137047d777ddff3e61c0fd0a. --- EESSI-install-software.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 2d8152ece9..0b45ad018d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -252,11 +252,6 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -# If $EESSI_DEV_PROJECT is set, we use EESSI-extend to install to -# /cvmfs/dev.eessi.io/ -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - $EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections @@ -298,6 +293,15 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi +# If in dev.eessi.io, allow building on top of software.eessi.io +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all + # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io + EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" + module use $EASYBUILD_INSTALLPATH/modules/all +fi + if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else From e5517b6db604092a0b5418de1066b22c1a5e88aa Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Feb 2025 22:09:11 +0100 Subject: [PATCH 062/112] Try again EESSI_PROJECT_INSTALL; temp override EESSI_CVMFS_REPO --- EESSI-install-software.sh | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0b45ad018d..05acbf49a2 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -252,8 +252,19 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # Force override of $EESSI_SOFTWARE_PATH + ORIG_EESSI_SOFTWARE_PATH=$EESSI_SOFTWARE_PATH + EESSI_SOFTWARE_PATH=$EESSI_CVMFS_REPO +fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} +if [[ ! -z ${ORIG_EESSI_SOFTWARE_PATH} ]]; then + EESSI_SOFTWARE_PATH=$ORIG_EESSI_SOFTWARE_PATH + unset $ORIG_EESSI_SOFTWARE_PATH +fi + # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install @@ -293,14 +304,6 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi -# If in dev.eessi.io, allow building on top of software.eessi.io -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all - # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io - EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" - module use $EASYBUILD_INSTALLPATH/modules/all -fi if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" From 6c9e295f14c49bb7549b7c2cdc38df83ee7a8729 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Feb 2025 22:23:07 +0100 Subject: [PATCH 063/112] Also unset $EESSI_CVMFS_INSTALL --- EESSI-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 05acbf49a2..fc30fd6281 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -253,6 +253,7 @@ export EESSI_CVMFS_INSTALL=1 # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + unset $EESSI_CVMFS_INSTALL EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} # Force override of $EESSI_SOFTWARE_PATH ORIG_EESSI_SOFTWARE_PATH=$EESSI_SOFTWARE_PATH From b71e2f9703d70187f3826cb5c7c88457a4fc767c Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Feb 2025 23:18:12 +0100 Subject: [PATCH 064/112] Fix typo in `unset` --- EESSI-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index fc30fd6281..02aa9abd4c 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -253,11 +253,12 @@ export EESSI_CVMFS_INSTALL=1 # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - unset $EESSI_CVMFS_INSTALL + unset EESSI_CVMFS_INSTALL EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} # Force override of $EESSI_SOFTWARE_PATH ORIG_EESSI_SOFTWARE_PATH=$EESSI_SOFTWARE_PATH EESSI_SOFTWARE_PATH=$EESSI_CVMFS_REPO + echo "$ORIG_EESSI_SOFTWARE_PATH temp override to $EESSI_SOFTWARE_PATH" fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 921145140cfa637e647ecf222053b91d1d2cfba7 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Feb 2025 23:38:13 +0100 Subject: [PATCH 065/112] Need to export EESSI_PROJECT_INSTALL --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 02aa9abd4c..276828e67f 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -254,7 +254,7 @@ export EESSI_CVMFS_INSTALL=1 export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then unset EESSI_CVMFS_INSTALL - EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} # Force override of $EESSI_SOFTWARE_PATH ORIG_EESSI_SOFTWARE_PATH=$EESSI_SOFTWARE_PATH EESSI_SOFTWARE_PATH=$EESSI_CVMFS_REPO From 48c61ef5bb2e2d081722149ee1371378fb42aad3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 12 Feb 2025 16:14:14 +0100 Subject: [PATCH 066/112] Again, use previous apprach --- EESSI-install-software.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 276828e67f..0b45ad018d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -252,21 +252,8 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - # Force override of $EESSI_SOFTWARE_PATH - ORIG_EESSI_SOFTWARE_PATH=$EESSI_SOFTWARE_PATH - EESSI_SOFTWARE_PATH=$EESSI_CVMFS_REPO - echo "$ORIG_EESSI_SOFTWARE_PATH temp override to $EESSI_SOFTWARE_PATH" -fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} -if [[ ! -z ${ORIG_EESSI_SOFTWARE_PATH} ]]; then - EESSI_SOFTWARE_PATH=$ORIG_EESSI_SOFTWARE_PATH - unset $ORIG_EESSI_SOFTWARE_PATH -fi - # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install @@ -306,6 +293,14 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi +# If in dev.eessi.io, allow building on top of software.eessi.io +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all + # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io + EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" + module use $EASYBUILD_INSTALLPATH/modules/all +fi if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" From ee2bcb13b2c6a7dad50d53f73a8abc9ebadf7a99 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 14 Feb 2025 19:25:44 +0100 Subject: [PATCH 067/112] Handle dev.eessi.io via EESSI_extend-easybuild.eb --- EESSI-extend-easybuild.eb | 12 +++++++++++- EESSI-install-software.sh | 18 +++++++++--------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index f74f36aca7..3a984da77d 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -107,6 +107,7 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then else -- Deal with user and project installs project_install = os.getenv("EESSI_PROJECT_INSTALL") + eessi_dev_project = os.getenv("EESSI_DEV_INSTALL") project_modulepath = nil if (project_install ~= nil) then -- Check the folder exists @@ -116,7 +117,16 @@ else if (mode() == "load") then LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) end - easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) + if (eessi_dev_project ~= nil) then + -- Project install is assumed to be $EESSI_CVMFS_REPO_OVERRIDE set to dev.eessi.io + eessi_software_path = os.getenv("EESSI_SOFTWARE_PATH") + -- Extract ${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} from ${EESSI_SOFTWARE_PATH} + os_type_software_subdir = string.match(eessi_software_path, "software/(.*)") + -- Construct installpath for dev.eessi.io and include project subdir + easybuild_installpath = pathJoin(project_install, 'versions', version, eessi_dev_project, 'software', os_type_software_subdir) + else + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) + end project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') end user_install = os.getenv("EESSI_USER_INSTALL") diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0b45ad018d..a2ed45b4cc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -252,6 +252,15 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + +# If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + # module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all + # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io + export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} + echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" +fi + source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} # Install full CUDA SDK and cu* libraries in host_injections @@ -293,15 +302,6 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi -# If in dev.eessi.io, allow building on top of software.eessi.io -if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all - # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io - EASYBUILD_INSTALLPATH=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - echo ">> \$EASYBUILD_INSTALLPATH overwritten to ${EASYBUILD_INSTALLPATH}" - module use $EASYBUILD_INSTALLPATH/modules/all -fi - if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else From e012bd4d83b0e61cc0445de7a1c3e3507db0d1d1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 17 Feb 2025 16:02:26 +0100 Subject: [PATCH 068/112] Try installing to dev.eessi.io/project/version --- EESSI-extend-easybuild.eb | 13 +------------ EESSI-install-software.sh | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index 3a984da77d..29a81acfb1 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -106,8 +106,6 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH") else -- Deal with user and project installs - project_install = os.getenv("EESSI_PROJECT_INSTALL") - eessi_dev_project = os.getenv("EESSI_DEV_INSTALL") project_modulepath = nil if (project_install ~= nil) then -- Check the folder exists @@ -117,16 +115,7 @@ else if (mode() == "load") then LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) end - if (eessi_dev_project ~= nil) then - -- Project install is assumed to be $EESSI_CVMFS_REPO_OVERRIDE set to dev.eessi.io - eessi_software_path = os.getenv("EESSI_SOFTWARE_PATH") - -- Extract ${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} from ${EESSI_SOFTWARE_PATH} - os_type_software_subdir = string.match(eessi_software_path, "software/(.*)") - -- Construct installpath for dev.eessi.io and include project subdir - easybuild_installpath = pathJoin(project_install, 'versions', version, eessi_dev_project, 'software', os_type_software_subdir) - else - easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) - end + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') end user_install = os.getenv("EESSI_USER_INSTALL") diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index a2ed45b4cc..ba95b2e6c1 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -257,7 +257,7 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then # module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io - export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} + export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE}/${EESSI_DEV_PROJECT} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From 367e30b136c79b78dbcb920e6fa18117ad379fcf Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 17 Feb 2025 17:19:23 +0100 Subject: [PATCH 069/112] Unset EESSI_CVMFS_INSTALL --- EESSI-install-software.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index ba95b2e6c1..20920e453e 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,8 +255,8 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - # module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all - # Overwrite EASYBUILD_INSTALLPATH to point to dev.eessi.io + # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_SITE_INSTALL + unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE}/${EESSI_DEV_PROJECT} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" fi From e17d4f3b361f57c2dff94b072e31a9467d681233 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Tue, 18 Feb 2025 10:34:41 +0100 Subject: [PATCH 070/112] Create installdir if it doesn't exist Co-authored-by: ocaisa --- EESSI-install-software.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 20920e453e..faedc60725 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -259,6 +259,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE}/${EESSI_DEV_PROJECT} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + mkdir -p ${EESSI_PROJECT_INSTALL} fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 2b96be5e5f98778354b0f26da283a0fe57eadafc Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 18 Feb 2025 12:51:49 +0100 Subject: [PATCH 071/112] Correct place for eessi_dev_project in software_dir in create_tarball.sh --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0b6819e81a..c565076674 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -33,7 +33,7 @@ if [ -z ${install_prefix_dir} ]; then fi # software_dir includes $eessi_dev_project if defined, otherwise install omit -software_dir="${cvmfs_repo}/versions/${eessi_version}${eessi_dev_project:+/$eessi_dev_project}/software/${os}/${cpu_arch_subdir}" +software_dir="${cvmfs_repo}${eessi_dev_project:+/$eessi_dev_project}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 From 82da386f0c96a99f9d3a63739e5d1aad13e75362 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Feb 2025 10:37:41 +0100 Subject: [PATCH 072/112] Restore project_install definition --- EESSI-extend-easybuild.eb | 1 + 1 file changed, 1 insertion(+) diff --git a/EESSI-extend-easybuild.eb b/EESSI-extend-easybuild.eb index 29a81acfb1..f74f36aca7 100644 --- a/EESSI-extend-easybuild.eb +++ b/EESSI-extend-easybuild.eb @@ -106,6 +106,7 @@ elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then easybuild_installpath = os.getenv("EESSI_SITE_SOFTWARE_PATH") else -- Deal with user and project installs + project_install = os.getenv("EESSI_PROJECT_INSTALL") project_modulepath = nil if (project_install ~= nil) then -- Check the folder exists From acb896a163118d103e62e65566e052ce05c8ccc6 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Feb 2025 16:35:43 +0100 Subject: [PATCH 073/112] Add $EESSI_DEV_PROJECT to CVMFS_REPO_OVERRIDE if available --- bot/build.sh | 2 +- create_tarball.sh | 27 +++++++++------------------ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 875644e84d..025f67dc14 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -142,7 +142,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME} +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/create_tarball.sh b/create_tarball.sh index c565076674..76e651d29b 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -20,20 +20,11 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults -cvmfs_repo=${EESSI_CVMFS_REPO} +# EESSI_CVMFS_REPO is followed by $EESSI_DEV_PROJECT if defined +cvmfs_repo=${EESSI_CVMFS_REPO}${eessi_dev_project:+/$eessi_dev_project} -# install to $eessi_version/$eessi_dev_project if defined, otherwise install -# to $eessi_version -install_prefix_dir=${eessi_version}${eessi_dev_project:+/$eessi_dev_project} -echo "Setting Install prefix directory to: ${install_prefix_dir}" +software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" -if [ -z ${install_prefix_dir} ]; then - echo "\${install_prefix_dir} is empty?!" >&2 - exit 2 -fi - -# software_dir includes $eessi_dev_project if defined, otherwise install omit -software_dir="${cvmfs_repo}${eessi_dev_project:+/$eessi_dev_project}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 @@ -74,18 +65,18 @@ fi # consider both CPU-only and accelerator subdirectories for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do - if [ -d ${install_prefix_dir}/software/${os}/${subdir}/modules ]; then + if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files - find ${install_prefix_dir}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module symlinks - find ${install_prefix_dir}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module files and symlinks - find ${install_prefix_dir}/software/${os}/${subdir}/modules/all -type f -o -type l \ + find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ >> ${module_files_list} fi - if [ -d ${install_prefix_dir}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then + if [ -d ${eessi_version}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then # installation directories but only those for which module files were created # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' # using EasyBuild's standard module naming scheme) match the name of the @@ -94,7 +85,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do # installation directories), the procedure will likely not work. for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" - ls -d ${install_prefix_dir}/software/${os}/${subdir}/software/${package_version} \ + ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match done fi From 4a8bceec8a5d1f806af14cf7f52d7303ee96e38b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Feb 2025 16:42:20 +0100 Subject: [PATCH 074/112] No longer need EESSI_DEV_PROJECT in minimal_eessi_env --- init/minimal_eessi_env | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 5150d49259..bc13a43bc4 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,16 +8,8 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults -# If $EESSI_DEV_PROJECT is defined, then we are building for dev.eessi.io -# and add a subdirectory to $EESSI_PREFIX -if [ -z ${EESSI_DEV_PROJECT} ]; then - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION -else - echo "\$EESSI_DEV_PROJECT: ${EESSI_DEV_PROJECT}; building for dev.eessi.io" - export EESSI_PREFIX=$EESSI_CVMFS_REPO_OVERRIDE/versions/$EESSI_VERSION/$EESSI_DEV_PROJECT - # Set EESSI_PREFIX to the dev project path for EESSI-extend - export EESSI_CVMFS_INSTALL=$EESSI_PREFIX -fi + +export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION if [[ $(uname -s) == 'Linux' ]]; then From 50937f797c07bff7a7285048e7d13fa4ea91b5d3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Feb 2025 17:07:28 +0100 Subject: [PATCH 075/112] EESSI_PROJECT_INSTALL should now be EESSI_CVMFS_REPO_OVERRIDE --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index faedc60725..9e2ccc0f94 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -257,7 +257,7 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_SITE_INSTALL unset EESSI_CVMFS_INSTALL - export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE}/${EESSI_DEV_PROJECT} + export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" mkdir -p ${EESSI_PROJECT_INSTALL} fi From a5c2cf30ea60ebb953230d8378bba01bfb7236cf Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Feb 2025 17:22:54 +0100 Subject: [PATCH 076/112] In dev.eessi.io EESSI_CVMFS_REPO now contains project subdir This is no longer necessary here then --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 025f67dc14..4c60b8f7bb 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -303,8 +303,8 @@ export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" \"${EESSI_DEV_PROJECT}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" "${EESSI_DEV_PROJECT}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From d0c063039e4a87101d1c7fb671bea25cbcccad01 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Feb 2025 17:31:51 +0100 Subject: [PATCH 077/112] Fix input checking --- create_tarball.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 76e651d29b..a6ba994124 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -4,24 +4,22 @@ set -e base_dir=$(dirname $(realpath $0)) -if [ $# -ne 6 ]; then - echo "ERROR: Usage: $0 (example: ESPREsSo) " >&2 +if [ $# -ne 5 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 accel_subdir=$4 -eessi_dev_project=$5 -target_tgz=$6 +target_tgz=$5 tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults -# EESSI_CVMFS_REPO is followed by $EESSI_DEV_PROJECT if defined -cvmfs_repo=${EESSI_CVMFS_REPO}${eessi_dev_project:+/$eessi_dev_project} +cvmfs_repo=${EESSI_CVMFS_REPO} software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" From 9dab8970e18b312a4d2298633de89335814afcad Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 28 Feb 2025 14:42:33 +0100 Subject: [PATCH 078/112] Use EESSI_DEV_PROJECT to eessi_container for upper-layer subdirs --- eessi_container.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index fc97f9877c..6a4a25c1d2 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -731,6 +731,13 @@ do cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi + # if building for dev.eessi.io, also include the project subdirectory + # so we redefine cvmfs_repo_name} to match EESSI_CVMFS_REPO_OVERRIDE + if [[ ${cvmfs_repo_name} == "dev.eessi.io" ]]; then + echo "Building for dev.eessi.io, appending " + cvmfs_repo_name = ${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + fi + # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} From cd6ca79ea5c9cbc38e4111e10f7ed7ea6094a896 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 28 Feb 2025 16:44:34 +0100 Subject: [PATCH 079/112] Export EESSI_DEV_PROJECT to eessi_container --- EESSI-install-software.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 9e2ccc0f94..b3b9e7165b 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -260,6 +260,8 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" mkdir -p ${EESSI_PROJECT_INSTALL} + # Need to export EESSI_DEV_PROJECT for eessi_container.sh + export ${EESSI_DEV_PROJECT} fi source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} From 7b4eef683b53be8ae91cd318df2e3056d9ecbb76 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 28 Feb 2025 16:46:05 +0100 Subject: [PATCH 080/112] Describe cvmfs_repo_name change in log --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 6a4a25c1d2..56f25b4efa 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -734,7 +734,7 @@ do # if building for dev.eessi.io, also include the project subdirectory # so we redefine cvmfs_repo_name} to match EESSI_CVMFS_REPO_OVERRIDE if [[ ${cvmfs_repo_name} == "dev.eessi.io" ]]; then - echo "Building for dev.eessi.io, appending " + echo "Building for dev.eessi.io, appending EESSI_DEV_PROJET to cvmfs_repo_name" cvmfs_repo_name = ${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} fi From bd44aed29d3cb3046fb3232c332d923e94228821 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 28 Feb 2025 17:11:09 +0100 Subject: [PATCH 081/112] Fix syntax --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 56f25b4efa..4882e2fce6 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -734,8 +734,8 @@ do # if building for dev.eessi.io, also include the project subdirectory # so we redefine cvmfs_repo_name} to match EESSI_CVMFS_REPO_OVERRIDE if [[ ${cvmfs_repo_name} == "dev.eessi.io" ]]; then - echo "Building for dev.eessi.io, appending EESSI_DEV_PROJET to cvmfs_repo_name" - cvmfs_repo_name = ${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + echo "Building for dev.eessi.io, appending EESSI_DEV_PROJECT: $EESSI_DEV_PROJECT to cvmfs_repo_name" + cvmfs_repo_name=${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} fi # always create a directory for the repository (e.g., to store settings, ...) From 2f8cb06a718fffd8f393670232e70244e6732768 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Mar 2025 09:43:03 +0100 Subject: [PATCH 082/112] Try creating directory earlier --- EESSI-install-software.sh | 1 + eessi_container.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index b3b9e7165b..92dd55131a 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -259,6 +259,7 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" + # Consider removing this (created in eessi_container.sh) mkdir -p ${EESSI_PROJECT_INSTALL} # Need to export EESSI_DEV_PROJECT for eessi_container.sh export ${EESSI_DEV_PROJECT} diff --git a/eessi_container.sh b/eessi_container.sh index 4882e2fce6..41090483e9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -736,6 +736,7 @@ do if [[ ${cvmfs_repo_name} == "dev.eessi.io" ]]; then echo "Building for dev.eessi.io, appending EESSI_DEV_PROJECT: $EESSI_DEV_PROJECT to cvmfs_repo_name" cvmfs_repo_name=${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + mkdir -p ${EESSI_PROJECT_INSTALL} fi # always create a directory for the repository (e.g., to store settings, ...) From 3f204043108abc1f209ac012dae78b5bfcfa574c Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Mar 2025 09:56:40 +0100 Subject: [PATCH 083/112] Make directory from correct variable --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 41090483e9..70718a9073 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -736,7 +736,7 @@ do if [[ ${cvmfs_repo_name} == "dev.eessi.io" ]]; then echo "Building for dev.eessi.io, appending EESSI_DEV_PROJECT: $EESSI_DEV_PROJECT to cvmfs_repo_name" cvmfs_repo_name=${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} - mkdir -p ${EESSI_PROJECT_INSTALL} + mkdir -p ${cvmfs_repo_name} fi # always create a directory for the repository (e.g., to store settings, ...) From 66738b3c1b0280afd97deb255200fa0aaad7bff8 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Mar 2025 14:25:37 +0100 Subject: [PATCH 084/112] Adjust only overlay subdirs, mount cvmfs repo correctly (no subdir allowed) --- eessi_container.sh | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 70718a9073..03e0794388 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -731,13 +731,6 @@ do cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi - # if building for dev.eessi.io, also include the project subdirectory - # so we redefine cvmfs_repo_name} to match EESSI_CVMFS_REPO_OVERRIDE - if [[ ${cvmfs_repo_name} == "dev.eessi.io" ]]; then - echo "Building for dev.eessi.io, appending EESSI_DEV_PROJECT: $EESSI_DEV_PROJECT to cvmfs_repo_name" - cvmfs_repo_name=${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} - mkdir -p ${cvmfs_repo_name} - fi # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} @@ -798,8 +791,8 @@ do EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-work" EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" export EESSI_WRITABLE_OVERLAY From 5b81fa9af6903fbf6bb721cab396093d841c4d0d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Mar 2025 15:21:56 +0100 Subject: [PATCH 085/112] mkdir container tmpdirs --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 03e0794388..2e3a1d5a1c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -780,8 +780,8 @@ do fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-work [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container From 54e4199673e1f41babc2afaa1d71fcb370c6ac9b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Mar 2025 15:28:59 +0100 Subject: [PATCH 086/112] dev project subdir in overlay_upper_dir --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index a6ba994124..e07def9ef2 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -29,7 +29,7 @@ if [ ! -d ${software_dir} ]; then fi cvmfs_repo_name=${cvmfs_repo#/cvmfs/} -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper" +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From 7c0db3f7b9bc3c656629d13f2004ffa6b340a60c Mon Sep 17 00:00:00 2001 From: Neves-P Date: Mon, 3 Mar 2025 15:51:09 +0100 Subject: [PATCH 087/112] Subdir needs to be added inside overlay --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index e07def9ef2..38135ec1b4 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -29,7 +29,7 @@ if [ ! -d ${software_dir} ]; then fi cvmfs_repo_name=${cvmfs_repo#/cvmfs/} -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper" +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From d6d2f1b31be611d2cc11416f87108962862733de Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 4 Mar 2025 15:40:04 +0100 Subject: [PATCH 088/112] Fix overlay_upper_dir in create_tarball --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 38135ec1b4..0014ba3e0a 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -29,7 +29,7 @@ if [ ! -d ${software_dir} ]; then fi cvmfs_repo_name=${cvmfs_repo#/cvmfs/} -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From dbc0c82abfcf8a18e28a2786706dc26b7af069c1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 14 Mar 2025 17:58:36 +0100 Subject: [PATCH 089/112] Remove repeated project name --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0014ba3e0a..38135ec1b4 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -29,7 +29,7 @@ if [ ! -d ${software_dir} ]; then fi cvmfs_repo_name=${cvmfs_repo#/cvmfs/} -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then From a0f2570192089a3d691155ab6b8b1643bdb66375 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 14 Mar 2025 17:58:57 +0100 Subject: [PATCH 090/112] Fix comment --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 92dd55131a..f391d1817f 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -255,7 +255,7 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_SITE_INSTALL + # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_PROJECT_INSTALL unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" From e481b17e2609effef18e7d750280cc8aa1496ec7 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 19 Mar 2025 09:47:36 +0100 Subject: [PATCH 091/112] Inspect script looks in project subdir --- bot/inspect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/inspect.sh b/bot/inspect.sh index 533968bffc..6ffd227b59 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -209,7 +209,7 @@ echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/$(cfg_get_value 'repository' 'repo_name')" +export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/$(cfg_get_value 'repository' 'repo_name')${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" echo "bot/inspect.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} From f3a45f2df8ab6c103a11e733b41657ad2af2eab2 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 19 Mar 2025 17:06:39 +0100 Subject: [PATCH 092/112] Remove project subdir before overaly-upper --- create_tarball.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 38135ec1b4..a278aca787 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -28,7 +28,9 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -cvmfs_repo_name=${cvmfs_repo#/cvmfs/} +echo $cvmfs_repo | grep -o -P '(?<=/cvmfs/).*(?=/)' +cvmfs_repo_name=${cvmfs_repo%"${EESSI_DEV_PROJECT}"} +cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" From 76c19fcb9d56b8fa5de6fdbf648295e1bf05ab18 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 19 Mar 2025 17:24:46 +0100 Subject: [PATCH 093/112] Add missing / --- create_tarball.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index a278aca787..7cf140650d 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -28,8 +28,7 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -echo $cvmfs_repo | grep -o -P '(?<=/cvmfs/).*(?=/)' -cvmfs_repo_name=${cvmfs_repo%"${EESSI_DEV_PROJECT}"} +cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" From dd52011c941e2c592977276a1b513c1d7555f7b4 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Mar 2025 14:56:08 +0100 Subject: [PATCH 094/112] Correct cvmfs_repo_name in container script --- eessi_container.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 2e3a1d5a1c..aca68a194f 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -730,7 +730,9 @@ do cfg_repo_id=${cvmfs_repo_name} cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi - + # remove project subdir in container + cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} + cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} From ece81279543b2e9de06c6c14b705fe2bf7a9ba02 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 20 Mar 2025 16:57:45 +0100 Subject: [PATCH 095/112] Leave rest of cvmfs_repo_name untouched Only remove EESSI_DEV_PROJECT --- eessi_container.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index aca68a194f..7aa7b979d2 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -731,8 +731,7 @@ do cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi # remove project subdir in container - cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} - cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} + cvmfs_repo_name=${cvmfs_repo_name%"/${EESSI_DEV_PROJECT}"} # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} From 9d980f9a93195cbf78f7f7abd85e603448b74feb Mon Sep 17 00:00:00 2001 From: Neves-P Date: Fri, 21 Mar 2025 17:10:57 +0100 Subject: [PATCH 096/112] EESSI_DEV_PROJECT comes after overlays --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 7aa7b979d2..65e83075fb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -781,8 +781,8 @@ do fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-work + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container From e98ce2bbdb6e9584e17c19b2b7772438d0d5a00c Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 17:17:47 +0100 Subject: [PATCH 097/112] Fix overlay paths in container --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 65e83075fb..f215c93ebc 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -792,8 +792,8 @@ do EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" export EESSI_WRITABLE_OVERLAY From 16c77b5fab3d7a7c89e0b6e50f689e2693afed2e Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 17:44:22 +0100 Subject: [PATCH 098/112] Print debug output --- eessi_container.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index f215c93ebc..8cca65fc77 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -732,9 +732,12 @@ do fi # remove project subdir in container cvmfs_repo_name=${cvmfs_repo_name%"/${EESSI_DEV_PROJECT}"} + echo "Adjusted cvmfs_repo_name: ${cvmfs_repo_name}" # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} + echo "Created directory for repository: ${EESSI_TMPDIR}/${cvmfs_repo_name}" + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) if [[ ${cvmfs_repo_access} == "ro" ]] ; then From 38f129cdebedfe03af014c86d7b799c7973b9def Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 18:19:17 +0100 Subject: [PATCH 099/112] Don't create child directory --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 8cca65fc77..add0190499 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -784,8 +784,8 @@ do fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container From 82d3828b52f5e917e12dcfc7ed433967bbda49a0 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 18:31:21 +0100 Subject: [PATCH 100/112] Create child dir, more debug output --- create_tarball.sh | 4 +++- eessi_container.sh | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 7cf140650d..f175e43c83 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -20,8 +20,9 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} - +echo "create_tarball.sh: cvmfs_repo=${cvmfs_repo}" software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" +echo "create_tarball.sh: software_dir=${software_dir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 @@ -31,6 +32,7 @@ fi cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" +echo "create_tarball.sh: overlay_upper_dir=${overlay_upper_dir}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then diff --git a/eessi_container.sh b/eessi_container.sh index add0190499..8cca65fc77 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -784,8 +784,8 @@ do fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container From 013d9424d1f39e39b6beaae0c63b5575dcde90d3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 19:01:01 +0100 Subject: [PATCH 101/112] Print created overlay dir in container --- eessi_container.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 8cca65fc77..52dbf9f466 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -786,6 +786,8 @@ do # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + echo "Created overlay directories ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container From 3758ecd47778e7eb3728e2cdd3e299fb948b3483 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 19:06:29 +0100 Subject: [PATCH 102/112] =?UTF-8?q?Create=20dir,=20but=20don=C2=B4t=20add?= =?UTF-8?q?=20subdir=20to=20envvar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- eessi_container.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 52dbf9f466..d3fc1cac82 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -787,7 +787,7 @@ do mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} echo "Created overlay directories ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" - + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container @@ -797,8 +797,8 @@ do EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" export EESSI_WRITABLE_OVERLAY From 38b6b07c86ba9402b2b441614bbc5db2c521766c Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 19:28:37 +0100 Subject: [PATCH 103/112] No special handling of tarball path in check-build --- bot/check-build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/check-build.sh b/bot/check-build.sh index faa2ff1e05..2f81c58758 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -482,9 +482,9 @@ if [[ ! -z ${TARBALL} ]]; then # if building for dev.eessi.io is set, the prefix is different # it needs the project name in the path - if [ ! -z ${EESSI_DEV_PROJECT} ]; then - prefix="${repo_version}/${EESSI_DEV_PROJECT}/software/${os_type}/${software_subdir}" - fi + # if [ ! -z ${EESSI_DEV_PROJECT} ]; then + # prefix="${repo_version}/${EESSI_DEV_PROJECT}/software/${os_type}/${software_subdir}" + # fi # if we build for an accelerator, the prefix is different if [[ ! -z ${accelerator} ]]; then From d0b5df652ef425dd75466c63f3c346439cccd80f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Sun, 23 Mar 2025 19:44:30 +0100 Subject: [PATCH 104/112] Cleanup --- EESSI-install-software.sh | 1 - bot/check-build.sh | 6 ------ create_tarball.sh | 4 ---- eessi_container.sh | 3 --- init/minimal_eessi_env | 2 -- 5 files changed, 16 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f391d1817f..090c825a36 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -207,7 +207,6 @@ export PYTHONPYCACHEPREFIX=$TMPDIR/pycache # - .lmod/lmodrc.lua # - .lmod/SitePackage.lua _eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -echo "_eessi_software_path: ${_eessi_software_path}" _lmod_cfg_dir=${_eessi_software_path}/.lmod _lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua if [ ! -f ${_lmod_rc_file} ]; then diff --git a/bot/check-build.sh b/bot/check-build.sh index 2f81c58758..41aeab258e 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -480,12 +480,6 @@ if [[ ! -z ${TARBALL} ]]; then accelerator=$(cfg_get_value "architecture" "accelerator") prefix="${repo_version}/software/${os_type}/${software_subdir}" - # if building for dev.eessi.io is set, the prefix is different - # it needs the project name in the path - # if [ ! -z ${EESSI_DEV_PROJECT} ]; then - # prefix="${repo_version}/${EESSI_DEV_PROJECT}/software/${os_type}/${software_subdir}" - # fi - # if we build for an accelerator, the prefix is different if [[ ! -z ${accelerator} ]]; then prefix="${prefix}/accel/${accelerator}" diff --git a/create_tarball.sh b/create_tarball.sh index f175e43c83..6115865367 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -20,10 +20,7 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} -echo "create_tarball.sh: cvmfs_repo=${cvmfs_repo}" software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" -echo "create_tarball.sh: software_dir=${software_dir}" - if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 @@ -32,7 +29,6 @@ fi cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" -echo "create_tarball.sh: overlay_upper_dir=${overlay_upper_dir}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then diff --git a/eessi_container.sh b/eessi_container.sh index d3fc1cac82..9de2cb37e9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -732,11 +732,9 @@ do fi # remove project subdir in container cvmfs_repo_name=${cvmfs_repo_name%"/${EESSI_DEV_PROJECT}"} - echo "Adjusted cvmfs_repo_name: ${cvmfs_repo_name}" # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} - echo "Created directory for repository: ${EESSI_TMPDIR}/${cvmfs_repo_name}" # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) @@ -786,7 +784,6 @@ do # use repo-specific overlay directories mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} - echo "Created overlay directories ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index bc13a43bc4..5e513c3c9f 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -8,10 +8,8 @@ EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # script takes *_OVERRIDEs into account source ${EESSI_INIT_DIR_PATH}/eessi_defaults - export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION - if [[ $(uname -s) == 'Linux' ]]; then export EESSI_OS_TYPE='linux' else From 74ea9ecf5fe32b8fcac38399606f51643b4aaab0 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 25 Mar 2025 11:42:21 +0100 Subject: [PATCH 105/112] Handle .lmod files in dev.eessi.io builds --- EESSI-install-software.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 090c825a36..c4c74b62b7 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -254,6 +254,8 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + # We keep track of the old install path so the SitePackage.lua and .lmodrc later + EASYBUILD_INSTALLPATH_STANDARD=${EASYBUILD_INSTALLPATH} # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_PROJECT_INSTALL unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} @@ -371,7 +373,13 @@ else done fi -export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + # Make sure .lmod files are not checked for dev.eeessi.io + export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" +else + export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" +fi + lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_rc_file From 8e3e010134a8c41c2a22defc8916412b99a9c10d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Tue, 25 Mar 2025 12:16:09 +0100 Subject: [PATCH 106/112] Handle LMOD_PACKAGE_PATH that was missing --- EESSI-install-software.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index c4c74b62b7..45f465fa58 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -376,8 +376,10 @@ fi if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then # Make sure .lmod files are not checked for dev.eeessi.io export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" + export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" else export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" + export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" fi lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" @@ -393,7 +395,6 @@ if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file" fi -export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_sitepackage_file From 4bb155910f196d1ab18992299874c2ba058d4f5f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 3 Apr 2025 15:51:17 +0200 Subject: [PATCH 107/112] Format tarball name --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 4c60b8f7bb..e944bf86d4 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -290,7 +290,7 @@ fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now source $software_layer_dir/init/eessi_defaults -export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'" From b314d96de191da1488e3bc40cf169925efaa961a Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 17 Apr 2025 17:06:03 +0200 Subject: [PATCH 108/112] Merge from main --- .github/workflows/scripts/compare_stacks.py | 131 ++++ .../workflows/scripts/compare_to_generic.sh | 26 + .github/workflows/test-software.eessi.io.yml | 15 +- .github/workflows/test_compare_stacks.yml | 48 ++ .github/workflows/tests_archdetect.yml | 1 + .../tests_link_nvidia_host_libraries.yml | 199 ++++++ EESSI-install-software.sh | 40 +- EESSI-remove-software.sh | 17 +- bot/build.sh | 44 +- bot/check-build.sh | 295 ++++---- bot/test.sh | 29 +- create_lmodsitepackage.py | 167 ++++- .../eessi-2023.06-eb-4.9.4-001-system.yml | 20 + ...0215-eb-4.9.4-CUDA-update-module-files.yml | 14 + .../eessi-2023.06-eb-4.9.4-2023a-CUDA.yml | 4 + .../eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 4 + .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 2 + .../eessi-2023.06-eb-4.9.4-001-system.yml | 9 + .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 14 + .../2023.06/eessi-2023.06-eb-4.9.4-2023b.yml | 20 + .../2023.06/eessi-2023.06-eb-5.0.0-2023a.yml | 6 + .../eessi-2023.06-eb-4.9.4-2023a-CUDA.yml | 20 + .../eessi-2023.06-eb-4.9.4-2023b-CUDA.yml | 25 + .../grace/eessi-2023.06-eb-4.8.2-2022b.yml | 11 + .../grace/eessi-2023.06-eb-4.9.2-2023a.yml | 6 + .../eessi-2023.06-eb-4.9.4-001-system.yml | 20 + .../grace/eessi-2023.06-eb-4.9.4-2022b.yml | 156 ++++ .../grace/eessi-2023.06-eb-4.9.4-2023a.yml | 210 ++++++ .../grace/eessi-2023.06-eb-4.9.4-2023b.yml | 224 ++++++ .../grace/eessi-2023.06-eb-5.0.0-2023a.yml | 5 + ...0241015-eb-4.9.4-LAMMPS-generic-builds.yml | 16 + ....12-rebuild-to-sync-exts-between-archs.yml | 6 + ...rom-hatchling-to-Python-sapphirerapids.yml | 15 + ...eb-4.9.2-Python-ctypes-sapphirerapids.yml} | 2 +- ...8.0-updated-easyconfig-sapphirerapids.yml} | 2 +- ...z3-with-python-bindings-sapphirerapids.yml | 8 + ...dle-Bioconductor-additional-extensions.yml | 9 + ...ikit-build-core-add-pyproject-metadata.yml | 8 + ...-rebuild-with-controlled-optimisations.yml | 8 + .../eessi-2023.06-eb-4.8.2-2022b.yml | 14 - .../eessi-2023.06-eb-4.9.0-2023a.yml | 12 - .../eessi-2023.06-eb-4.9.1-2023a.yml | 4 - .../eessi-2023.06-eb-4.9.2-2022b.yml | 2 - .../eessi-2023.06-eb-4.9.2-2023a.yml | 12 - .../eessi-2023.06-eb-4.9.2-2023b.yml | 3 - .../eessi-2023.06-eb-4.9.4-2022b.yml | 5 - .../eessi-2023.06-eb-4.8.2-001-system.yml | 0 .../eessi-2023.06-eb-4.8.2-2022b.yml | 17 + .../eessi-2023.06-eb-4.8.2-2023a.yml | 21 +- .../eessi-2023.06-eb-4.9.0-001-system.yml | 0 .../eessi-2023.06-eb-4.9.0-2022b.yml | 1 - .../eessi-2023.06-eb-4.9.0-2023a.yml | 86 +++ .../eessi-2023.06-eb-4.9.0-2023b.yml | 10 +- .../eessi-2023.06-eb-4.9.1-2022b.yml | 8 + .../eessi-2023.06-eb-4.9.1-2023a.yml | 19 + .../eessi-2023.06-eb-4.9.1-2023b.yml | 4 +- .../eessi-2023.06-eb-4.9.2-001-system.yml | 0 .../eessi-2023.06-eb-4.9.2-2022b.yml | 52 ++ .../eessi-2023.06-eb-4.9.2-2023a.yml | 86 +++ .../eessi-2023.06-eb-4.9.2-2023b.yml | 27 + .../eessi-2023.06-eb-4.9.3-2023a.yml | 8 + .../eessi-2023.06-eb-4.9.3-2023b.yml | 11 + .../eessi-2023.06-eb-4.9.4-001-system.yml | 0 .../eessi-2023.06-eb-4.9.4-2023a.yml | 53 ++ .../eessi-2023.06-eb-4.9.4-2023b.yml | 39 + .../zen4/eessi-2023.06-eb-4.8.2-2022b.yml | 11 + .../zen4/eessi-2023.06-eb-4.9.0-2022b.yml | 28 + .../zen4/eessi-2023.06-eb-4.9.1-2022b.yml | 8 + .../zen4/eessi-2023.06-eb-4.9.2-2022b.yml | 52 ++ eb_hooks.py | 81 ++- eessi-2023.06-known-issues.yml | 11 + eessi_container.sh | 63 +- init/arch_specs/eessi_arch_x86.spec | 11 +- init/eessi_environment_variables | 10 - init/modules/EESSI/2023.06.lua | 9 - install_apptainer_ubuntu.sh | 31 +- reframe_config_bot.py.tmpl | 57 -- run_tests.sh | 8 +- .../nvidia/install_cuda_host_injections.sh | 6 +- .../nvidia/link_nvidia_host_libraries.sh | 673 ++++++++++++------ test_suite.sh | 83 ++- .../AWS-Rocky8-8488C.all.output | 1 + .../sapphirerapids/AWS-Rocky8-8488C.cpuinfo | 26 + .../sapphirerapids/AWS-Rocky8-8488C.output | 1 + tests/nvidia-libs/mock-nvidia-libs.sh | 87 +++ tests/nvidia-libs/mock-nvidia-smi.sh | 35 + 86 files changed, 3012 insertions(+), 630 deletions(-) create mode 100644 .github/workflows/scripts/compare_stacks.py create mode 100755 .github/workflows/scripts/compare_to_generic.sh create mode 100644 .github/workflows/test_compare_stacks.yml create mode 100644 .github/workflows/tests_link_nvidia_host_libraries.yml create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-001-system.yml create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250215-eb-4.9.4-CUDA-update-module-files.yml create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-001-system.yml create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.8.2-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.2-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-001-system.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023b.yml create mode 100644 easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-5.0.0-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250212-eb-4.9.4-R-bundle-CRAN-2023.12-rebuild-to-sync-exts-between-archs.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250223-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python-sapphirerapids.yml rename easystacks/software.eessi.io/2023.06/rebuilds/{20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml => 20250224-eb-4.9.2-Python-ctypes-sapphirerapids.yml} (98%) rename easystacks/software.eessi.io/2023.06/rebuilds/{20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml => 20250228-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphirerapids.yml} (96%) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-rebuild-z3-with-python-bindings-sapphirerapids.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-R-bundle-Bioconductor-additional-extensions.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-scikit-build-core-add-pyproject-metadata.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20250410-eb-5.0.0-siesta-rebuild-with-controlled-optimisations.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml delete mode 100644 easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.8.2-001-system.yml (100%) create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2022b.yml rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.8.2-2023a.yml (84%) rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.9.0-001-system.yml (100%) rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.9.0-2022b.yml (92%) create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023a.yml rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.9.0-2023b.yml (85%) create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023a.yml rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.9.1-2023b.yml (76%) rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.9.2-001-system.yml (100%) create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023b.yml create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023b.yml rename easystacks/software.eessi.io/2023.06/{sapphire_rapids => sapphirerapids}/eessi-2023.06-eb-4.9.4-001-system.yml (100%) create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023a.yml create mode 100644 easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml delete mode 100644 reframe_config_bot.py.tmpl create mode 100644 tests/archdetect/x86_64/intel/sapphirerapids/AWS-Rocky8-8488C.all.output create mode 100644 tests/archdetect/x86_64/intel/sapphirerapids/AWS-Rocky8-8488C.cpuinfo create mode 100644 tests/archdetect/x86_64/intel/sapphirerapids/AWS-Rocky8-8488C.output create mode 100644 tests/nvidia-libs/mock-nvidia-libs.sh create mode 100644 tests/nvidia-libs/mock-nvidia-smi.sh diff --git a/.github/workflows/scripts/compare_stacks.py b/.github/workflows/scripts/compare_stacks.py new file mode 100644 index 0000000000..9e81f61538 --- /dev/null +++ b/.github/workflows/scripts/compare_stacks.py @@ -0,0 +1,131 @@ +import argparse +import os +import re +import glob +import json + + +def parse_module_file(module_file_path): + """ + Extracts module name, version, and extensions from a module file. + """ + module_name = os.path.basename(os.path.dirname(module_file_path)) + version = os.path.basename(module_file_path) + + try: + with open(module_file_path, "r") as file: + content = file.read() + + # Extract extensions from content using regex + match = re.search(r'extensions\("(.+)"\)', content) + extensions = [] + + if match: + # Split the list of packages by commas + packages = match.group(1) + for pkg in packages.split(","): + parts = pkg.split("/") + + # Check if the package is in the name/version format + if len(parts) == 2: + extensions.append((parts[0], parts[1])) + elif len(parts) == 1: + extensions.append((parts[0], "none")) + else: + print(f"Warning: Skipping invalid package format: {pkg}") + + return {(module_name, version): tuple(extensions)} + + except Exception as e: + print(f"Error parsing module file {module_file_path}: {e}") + return {(module_name, version): ()} + + +def get_available_modules(base_dir): + """ + Get the list of modules from all subdirectories inside the specified base directory. + """ + try: + modules = {} + # Only look for .lua files + for module_path in glob.glob(os.path.join(base_dir, "*/*.lua")): + modules.update(parse_module_file(module_path)) + return modules + + except Exception as e: + print(f"Error retrieving modules from {base_dir}: {e}") + return {} + + +def compare_stacks(dir1, dir2): + """ + Compare two sets of Lmod module files, including versions and extensions. + """ + modules1 = get_available_modules(dir1) + modules2 = get_available_modules(dir2) + + # Find differences between the two dictionaries + modules_removed = set(modules1.keys()) - set(modules2.keys()) + modules_added = set(modules2.keys()) - set(modules1.keys()) + matching_keys = set(modules1.keys()) & set(modules2.keys()) + + diff_results = { + "module_differences": { + "missing": list("/".join(module) for module in modules_removed), + "added": list("/".join(module) for module in modules_added), + }, + "extension_differences": [], + } + + # Compare extensions for matching keys + for key in matching_keys: + if modules1[key] != modules2[key]: + diff_results["extension_differences"].append( + { + "/".join(key): { + "missing": list( + "/".join(key) + for key in list(set(modules1[key]) - set(modules2[key])) + ), + "added": list( + "/".join(key) + for key in list(set(modules2[key]) - set(modules1[key])) + ), + } + } + ) + + return diff_results + + +def main(): + # Set up argument parser + parser = argparse.ArgumentParser(description="Compare two Lmod module directories") + parser.add_argument("path1", type=str, help="The first directory path") + parser.add_argument("path2", type=str, help="The second directory path") + + # Parse the arguments + args = parser.parse_args() + + # Validate the paths + for path in [args.path1, args.path2]: + if not os.path.exists(path): + print(f"Warning: Path does not exist: {path}") + + # Compare the stacks + diff_results = compare_stacks(args.path1, args.path2) + + # Print the differences + if any( + [ + diff_results["module_differences"]["missing"], + diff_results["module_differences"]["added"], + diff_results["extension_differences"], + ] + ): + print(json.dumps(diff_results, indent=2)) + exit(1) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/compare_to_generic.sh b/.github/workflows/scripts/compare_to_generic.sh new file mode 100755 index 0000000000..59a1397ec5 --- /dev/null +++ b/.github/workflows/scripts/compare_to_generic.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Take the arguments +base_dir=$1 +target_arch=$2 +modules_subdir="modules/all" +# Decide if we want x86_64 or aarch64 +arch=$(echo $target_arch | cut -d"/" -f1) +# Get the generic directory +source_of_truth="$arch/generic" +case $arch in + "x86_64") + echo "Using $source_of_truth as source of truth" + ;; + "aarch64") + echo "Using $source_of_truth as source of truth" + ;; + *) + echo "I don't understand the base architecture: $arch" + exit 1 + ;; +esac +source_of_truth_modules="$base_dir/$source_of_truth/$modules_subdir" +arch_modules="$base_dir/$target_arch/$modules_subdir" +echo "Comparing $arch_modules to $source_of_truth_modules" +python3 $script_dir/compare_stacks.py $source_of_truth_modules $arch_modules diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index f569c664c9..b8979a49aa 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -29,6 +29,7 @@ jobs: - x86_64/amd/zen3 - x86_64/amd/zen4 - x86_64/intel/haswell + - x86_64/intel/sapphirerapids - x86_64/intel/skylake_avx512 - x86_64/generic steps: @@ -59,20 +60,6 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)" for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then - if grep -q 2022b <<<"${easystack_file}"; then - # skip the check of installed software on zen4 for foss/2022b builds - continue - fi - if [[ $easystack_file == *"rebuilds"* ]]; then - # Also handle rebuilds, make a temporary EasyStack file where we clean out all 2022b stuff and use that - new_easystack=$(mktemp pruned_easystackXXX --suffix=.yml) - # first clean out the options then clean out the .eb name - sed '/2022b\|12\.2\.0/,/\.eb/{/\.eb/!d}' "${easystack_file}" | sed '/2022b\|12\.2\.0/d' > $new_easystack - diff --unified=0 "$easystack_file" "$new_easystack" || : - easystack_file="$new_easystack" - fi - fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? diff --git a/.github/workflows/test_compare_stacks.yml b/.github/workflows/test_compare_stacks.yml new file mode 100644 index 0000000000..c47f174705 --- /dev/null +++ b/.github/workflows/test_compare_stacks.yml @@ -0,0 +1,48 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Check for discrepencies between software stacks in software.eessi.io +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 +jobs: + compare_stacks: + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + EESSI_VERSION: + - 2023.06 + COMPARISON_ARCH: + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 + - x86_64/amd/zen2 + - x86_64/amd/zen3 + - x86_64/amd/zen4 + - x86_64/intel/haswell + - x86_64/intel/skylake_avx512 + - x86_64/intel/sapphirerapids + steps: + - name: Check out software-layer repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Mount EESSI CernVM-FS pilot repository + uses: eessi/github-action-eessi@v3 + + - name: Compare stacks + run: | + export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + env | grep ^EESSI | sort + + # Compare the requested architecture to the generic stack + # (assumes the general structure /cvmfs/software.eessi.io/versions/2023.06/software/linux/$COMPARISON_ARCH/modules/all) + .github/workflows/scripts/compare_to_generic.sh ${EESSI_PREFIX}/software/${EESSI_OS_TYPE} ${{matrix.COMPARISON_ARCH}} diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 82438d8e92..94fcf68d17 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -13,6 +13,7 @@ jobs: matrix: proc_cpuinfo: - x86_64/intel/haswell/archspec-linux-E5-2680-v3 + - x86_64/intel/sapphirerapids/AWS-Rocky8-8488C - x86_64/intel/skylake_avx512/archspec-linux-6132 - x86_64/amd/zen2/Azure-CentOS7-7V12 - x86_64/amd/zen3/Azure-CentOS7-7V73X diff --git a/.github/workflows/tests_link_nvidia_host_libraries.yml b/.github/workflows/tests_link_nvidia_host_libraries.yml new file mode 100644 index 0000000000..22ebc21084 --- /dev/null +++ b/.github/workflows/tests_link_nvidia_host_libraries.yml @@ -0,0 +1,199 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Test NVIDIA Host Libraries Linking +on: + push: + branches: + - '*-software.eessi.io' # Matches any branch ending with '-software.eessi.io' + pull_request: + paths: + - 'scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh' # PR changes only relevant for this specific file + - '.github/workflows/tests_link_nvidia_host_libraries.yml' # Also test when changing the tests themselves +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + build: + runs-on: ubuntu-24.04 + steps: + - name: checkout + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Initialize EESSI + uses: eessi/github-action-eessi@v3 + + - name: Setup mock NVIDIA libraries + run: | + # Run the script to create mock libraries + chmod +x ./tests/nvidia-libs/mock-nvidia-libs.sh + echo ">>> Running ./tests/nvidia-libs/mock-nvidia-libs.sh" + ./tests/nvidia-libs/mock-nvidia-libs.sh + + # Create symlink to override host's ldconfig, since the script tries to use /sbin/ldconfig first. + echo "Symlinking ldconfig to /sbin/ldconfig" + sudo ln -sf /tmp/ldconfig/ldconfig /sbin/ldconfig + + # Verify the symlink was created correctly + ls -la /sbin/ldconfig + + - name: Setup mock nvidia-smi + run: | + # Create directory for mock nvidia-smi + mkdir -p /tmp/nvidia-bin + + # Copy the mock script + chmod +x ./tests/nvidia-libs/mock-nvidia-smi.sh + echo ">>> Copying ./tests/nvidia-libs/mock-nvidia-smi.sh" + cp ./tests/nvidia-libs/mock-nvidia-smi.sh /tmp/nvidia-bin/nvidia-smi + + # Add to PATH + echo "Updating PATH" + echo "PATH=/tmp/nvidia-bin:$PATH" >> $GITHUB_ENV + + - name: Test LD_PRELOAD mode + run: | + echo ">>> Testing LD_PRELOAD mode" + + # Run the script with LD_PRELOAD option (shouldn't create symlinks) + output=$(./scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh --show-ld-preload || { echo "Script returned non-zero: $?"; echo $output; exit 1; }) + + echo "$output" + + echo ">>> Running checks" + + # Check for expected outputs + echo "$output" | grep "export EESSI_GPU_COMPAT_LD_PRELOAD=" || { echo "EESSI_GPU_COMPAT_LD_PRELOAD not found in output"; exit 1; } + echo "$output" | grep "export EESSI_GPU_LD_PRELOAD=" || { echo "EESSI_GPU_LD_PRELOAD not found in output"; exit 1; } + echo "$output" | grep "export EESSI_OVERRIDE_GPU_CHECK=" || { echo "EESSI_OVERRIDE_GPU_CHECK not found in output"; exit 1; } + + # Verify that no symlinks were created + if [ -e "/opt/eessi/nvidia/x86_64/host/driver_version.txt" ]; then + echo "Error: symlinks were created in LD_PRELOAD mode" + exit 1 + fi + + echo "LD_PRELOAD mode test passed." + + - name: Test normal run (first time) + run: | + echo ">>> Testing normal run - first time" + + # Run with verbose mode + output=$(./scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh --verbose || { echo "Script returned non-zero: $?"; echo $output; exit 1; }) + + echo "$output" + + echo ">>> Running checks" + + # Check if NVIDIA GPU was detected - Driver version and CUDA version are hardcoded in `tests/nvidia-libs/mock-nvidia-smi.sh` + echo "$output" | grep "Found NVIDIA GPU driver version 535.129.03" || { echo "Failed to detect NVIDIA driver version"; exit 1; } + echo "$output" | grep "Found host CUDA version 8.0" || { echo "Failed to detect CUDA version"; exit 1; } + + # Check if libraries were found + echo "$output" | grep "Matched.*CUDA Libraries" || { echo "Failed to match CUDA libraries"; exit 1; } + + # Verify symlinks were created + if [ ! -d "/opt/eessi/nvidia/x86_64/host" ]; then + echo "Error: host directory wasn't created" + exit 1 + fi + + # Check if version files were created + if [ ! -f "/opt/eessi/nvidia/x86_64/host/driver_version.txt" ]; then + echo "Error: driver_version.txt wasn't created" + exit 1 + fi + + # Check driver version content + grep "535.129.03" "/opt/eessi/nvidia/x86_64/host/driver_version.txt" || { echo "Incorrect driver version"; exit 1; } + + # Check if latest symlink was created + if [ ! -L "/opt/eessi/nvidia/x86_64/latest" ]; then + echo "Error: 'latest' symlink wasn't created" + exit 1 + fi + + # Check if latest points to host + readlink "/opt/eessi/nvidia/x86_64/latest" | grep "host" || { echo "latest doesn't point to host"; exit 1; } + + # Check if symlinks to libraries were created and point to correct files + echo ">>> Checking library symlinks" + + # List dir with libraries + echo "Showing content of /tmp/nvidia_libs" + echo "$(ls -l /tmp/nvidia_libs)" + echo "Showing content of /tmp/nvidia_libs_duplicate" + echo "$(ls -l /tmp/nvidia_libs_duplicate)" + echo "Showing content of /opt/eessi/nvidia/x86_64/host" + echo "$(ls -l /opt/eessi/nvidia/x86_64/host)" + + # List expected library names - list of libraries is hardcoded in `tests/nvidia-libs/mock-nvidia-libs.sh` + libraries=( + "libcuda.so" + "libcuda.so.1" + "libnvidia-ml.so" + "libnvidia-ml.so.1" + "libnvidia-ptxjitcompiler.so" + "libnvidia-ptxjitcompiler.so.1" + "libcudadebugger.so" + "libcudadebugger.so.1" + ) + + # Check each expected library symlink + for lib in "${libraries[@]}"; do + lib_path="/opt/eessi/nvidia/x86_64/host/$lib" + + # Check if the symlink exists + if [ ! -L "$lib_path" ]; then + echo "Error: Symlink for $lib was not created" + exit 1 + fi + + # Check if symlink target exists + target=$(readlink "$lib_path") + if [ ! -e "$target" ]; then + echo "Error: Symlink $lib_path points to non-existent file: $target" + exit 1 + fi + + # Verify it points to our mock library in /tmp/nvidia_libs or /tmp/nvidia_libs_duplicate + if [[ "$target" != "/tmp/nvidia_libs/$lib"* && "$target" != "/tmp/nvidia_libs_duplicate/$lib"* ]]; then + echo "Error: Symlink $lib_path points to $target, which is not in our mock directories" + exit 1 + fi + + echo ">>> Verified symlink: $lib -> $target" + done + + echo "First normal run test passed" + + - name: Test normal run (second time) + run: | + echo ">>> Testing normal run - second time - should be idempotent" + # Remove all write permissions on /opt/eessi so any attempts to write files fail + chmod -R a-w /opt/eessi + + # Store file timestamps before second run (ignoring access time) + stat_before=$(stat --format="%n %s %y %U %G %m %i" "/opt/eessi/nvidia/x86_64/host/driver_version.txt") + + # Run script again + output=$(./scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh || { echo "Script returned non-zero: $?"; echo $output; exit 1; }) + + echo "$output" + + echo ">>> Running checks" + + # Store file timestamps after second run (ignoring access time) + stat_after=$(stat --format="%n %s %y %U %G %m %i" "/opt/eessi/nvidia/x86_64/host/driver_version.txt") + + # Compare timestamps - should be the same (files shouldn't be modified) + if [[ "$stat_before" != "$stat_after" ]]; then + echo "Error: files were modified on second run when they shouldn't have been" + echo "Before: $stat_before" + echo "After: $stat_after" + exit 1 + fi + + # Check for message indicating that libraries are already linked + echo "$output" | grep "have already been linked" || { echo "Missing 'already linked' message"; exit 1; } + + echo "Second normal run test passed" + \ No newline at end of file diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 45f465fa58..4d303906bb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -53,6 +53,23 @@ function copy_build_log() { fi } +function safe_module_use { + # add a given non-empty directory to $MODULEPATH if and only if it is not yet in + directory=${1} + + if [[ -z ${directory+x} ]]; then + echo "safe_module_use: given directory unset or empty; not adding it to \$MODULEPATH (${MODULEPATH})" + return + fi + if [[ ":${MODULEPATH}:" == *":${directory}:"* ]]; then + echo "safe_module_use: directory '${directory}' is already in \$MODULEPATH (${MODULEPATH}); not adding it again" + return + else + echo "safe_module_use: directory '${directory}' is not yet in \$MODULEPATH (${MODULEPATH}); adding it" + module use ${directory} + fi +} + POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do @@ -266,7 +283,9 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then export ${EESSI_DEV_PROJECT} fi +echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} +echo "DEBUG: after loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" # Install full CUDA SDK and cu* libraries in host_injections # Hardcode this for now, see if it works @@ -286,11 +305,19 @@ fi # Install NVIDIA drivers in host_injections (if they exist) if command_exists "nvidia-smi"; then - echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..." - ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh + export LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}" + nvidia-smi --version + ec=$? + if [ ${ec} -eq 0 ]; then + echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..." + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh + else + echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully." + echo "This script now assumes this is NOT a GPU node." + echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error." + fi fi - if [ ! -z "${shared_fs_path}" ]; then shared_eb_sourcepath=${shared_fs_path}/easybuild/sources echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" @@ -307,6 +334,11 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi +echo "DEBUG: adding path '$EASYBUILD_INSTALLPATH/modules/all' to MODULEPATH='${MODULEPATH}'" +#module use $EASYBUILD_INSTALLPATH/modules/all +safe_module_use $EASYBUILD_INSTALLPATH/modules/all +echo "DEBUG: after adding module path // MODULEPATH='${MODULEPATH}'" + if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else @@ -373,6 +405,7 @@ else done fi +echo "DEBUG: before creating/updating lmod config files // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then # Make sure .lmod files are not checked for dev.eeessi.io export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" @@ -383,6 +416,7 @@ else fi lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" +echo "DEBUG: lmod_rc_file='${lmod_rc_file}'" if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_rc_file lmod_rc_file=$(echo ${lmod_rc_file} | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@") diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e4b377fd16..770b7e4117 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -116,9 +116,12 @@ if [ $EUID -eq 0 ]; then if [ -f ${easystack_file} ]; then echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." # we need to remove existing installation directories first, - # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # so let's figure out which modules have to be rebuilt by doing a + # dry-run and grepping "someapp/someversion" for the relevant + # lines (with [R] or [F]) + # * [F] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) - rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[[FR]\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') @@ -130,13 +133,13 @@ if [ $EUID -eq 0 ]; then echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} - # recreate the installation directories and first-level subdirectories to work around permission denied - # issues when rebuilding the package (see https://github.com/EESSI/software-layer/issues/556) + # recreate the installation directory and do an ls on the first-level subdirectories to work around + # permission issues when reinstalling the application (see https://github.com/EESSI/software-layer/issues/556) echo_yellow "Recreating an empty ${app_dir}..." mkdir -p ${app_dir} - for app_subdir in ${app_subdirs}; do - mkdir -p ${app_subdir} - done + # these subdirs don't (and shouldn't) exist, but we need to do the ls anyway as a workaround, + # so redirect to /dev/null and ignore the exit code + ls ${app_subdirs} >& /dev/null || true done else fatal_error "Easystack file ${easystack_file} not found!" diff --git a/bot/build.sh b/bot/build.sh index e944bf86d4..c0d9b03bc5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -90,11 +90,15 @@ if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR fi -echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " -# replace any env variable in ${LOCAL_TMP} with its -# current value (e.g., a value that is local to the job) -STORAGE=$(envsubst <<< ${LOCAL_TMP}) -echo "'${STORAGE}'" +if [[ -z "${TMPDIR}" ]]; then + echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " + # replace any env variable in ${LOCAL_TMP} with its + # current value (e.g., a value that is local to the job) + STORAGE=$(envsubst <<< ${LOCAL_TMP}) +else + STORAGE=${TMPDIR} +fi +echo "bot/build.sh: STORAGE='${STORAGE}'" # make sure ${STORAGE} exists mkdir -p ${STORAGE} @@ -115,7 +119,8 @@ mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then - for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + IFS=',' read -r -a modules <<< "$(echo "${LOAD_MODULES}")" + for mod in "${modules[@]}"; do echo "bot/build.sh: loading module '${mod}'" module load ${mod} @@ -179,6 +184,15 @@ if [[ "${REPOSITORY_NAME}" == "dev.eessi.io" ]]; then COMMON_ARGS+=("--repository" "software.eessi.io,access=ro") fi +# add $software_layer_dir and /dev as extra bind paths +# - $software_layer_dir is needed because it is used as prefix for running scripts +# - /dev is needed to access /dev/fuse +COMMON_ARGS+=("--extra-bind-paths" "${software_layer_dir},/dev") + +# pass through '--contain' to avoid leaking in scripts into the container session +# note, --pass-through can be used multiple times if needed +COMMON_ARGS+=("--pass-through" "--contain") + # make sure to use the same parent dir for storing tarballs of tmp PREVIOUS_TMP_DIR=${PWD}/previous_tmp @@ -243,14 +257,28 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} # prepare arguments to eessi_container.sh specific to build step BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") + # add options required to handle NVIDIA support if command_exists "nvidia-smi"; then - echo "Command 'nvidia-smi' found, using available GPU" - BUILD_STEP_ARGS+=("--nvidia" "all") + # Accept that this may fail + set +e + nvidia-smi --version + ec=$? + set -e + if [ ${ec} -eq 0 ]; then + echo "Command 'nvidia-smi' found, using available GPU" + BUILD_STEP_ARGS+=("--nvidia" "all") + else + echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully." + echo "This script now assumes this is NOT a GPU node." + echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error." + BUILD_STEP_ARGS+=("--nvidia" "install") + fi else echo "No 'nvidia-smi' found, no available GPU but allowing overriding this check" BUILD_STEP_ARGS+=("--nvidia" "install") fi + # Retain location for host injections so we don't reinstall CUDA # (Always need to run the driver installation as available driver may change) if [[ ! -z ${SHARED_FS_PATH} ]]; then diff --git a/bot/check-build.sh b/bot/check-build.sh index 41aeab258e..a3c3a3a1d2 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -8,6 +8,7 @@ # https://github.com/EESSI/software-layer.git # # author: Thomas Roeblitz (@trz42) +# author: Samuel Moors (@smoors) # # license: GPLv2 # @@ -59,10 +60,12 @@ display_help() { echo " OPTIONS:" echo " -h | --help - display this usage information [default: false]" echo " -v | --verbose - display more information [default: false]" + echo " --use-check-build-artefacts-script - alternative build artefacts check (sources file check-build-artefacts.sh if exists) [default: false]" } # set defaults for command line arguments VERBOSE=0 +USE_CHECK_BUILD_ARTEFACTS_SCRIPT=0 POSITIONAL_ARGS=() @@ -76,6 +79,10 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift 1 ;; + --use-check-build-artefacts-script) + USE_CHECK_BUILD_ARTEFACTS_SCRIPT=1 + shift 1 + ;; --) shift POSITIONAL_ARGS+=("$@") # save positional args @@ -157,20 +164,22 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" fi -TGZ=-1 -TARBALL= -if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_tgz_created="\.tar\.gz created!" - grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) - if [[ $? -eq 0 ]]; then - TGZ=1 - TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') - else - TGZ=0 - fi - # have to be careful to not add searched for pattern into slurm out file - [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then + TGZ=-1 + TARBALL= + if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_tgz_created="\.tar\.gz created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') + else + TGZ=0 + fi + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" + fi fi [[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" @@ -180,7 +189,9 @@ fi [[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" [[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" -[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then + [[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +fi # Here, we try to do some additional analysis on the output file # to see if we can print a more clear 'reason' for the failure @@ -208,8 +219,8 @@ if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \ [[ ${FAILED} -eq 0 ]] && \ [[ ${MISSING} -eq 0 ]] && \ [[ ${NO_MISSING} -eq 1 ]] && \ - [[ ${TGZ} -eq 1 ]] && \ - [[ ! -z ${TARBALL} ]]; then + [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || ${TGZ} -eq 1 ]] && \ + [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -ne 0 || -n ${TARBALL} ]]; then # SUCCESS status="SUCCESS" reason="" @@ -417,140 +428,150 @@ success_msg="found message(s) matching ${GP_no_missing}" failure_msg="no message matching ${GP_no_missing}" comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") -success_msg="found message matching ${GP_tgz_created}" -failure_msg="no message matching ${GP_tgz_created}" -comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then + success_msg="found message matching ${GP_tgz_created}" + failure_msg="no message matching ${GP_tgz_created}" + comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") +fi # Now, do the actual replacement of __DETAILS_FMT__ comment_details_fmt="
_Details_
__DETAILS_LIST__
" comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}" comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} -# first construct comment_artefacts_list -# then use it to set comment_artefacts -comment_artifacts_list="" - -# TARBALL should only contain a single tarball -if [[ ! -z ${TARBALL} ]]; then - # Example of the detailed information for a tarball. The actual result MUST be a - # single line (no '\n') or it would break the structure of the markdown table - # that holds status updates of a bot job. - # - #
- #
- # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz - # size: 234 MiB (245366784 bytes)
- # entries: 1234
- # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
- # 
-    #       GCC/9.3.0.lua

-    #       GCC/10.3.0.lua

-    #       OpenSSL/1.1.lua
-    #
- # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ - # 
-    #       GCC/9.3.0/

-    #       CMake/3.20.1-GCCcore-10.3.0/

-    #       OpenMPI/4.1.1-GCC-10.3.0/
-    #
- # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ - # 
-    #       .lmod/cache/spiderT.lua

-    #       .lmod/cache/spiderT.luac_5.1

-    #       .lmod/cache/timestamp
-    #
- #
- #
- size="$(stat --dereference --printf=%s ${TARBALL})" - size_mib=$((${size} >> 20)) - tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) - tar tf ${TARBALL} > ${tmpfile} - entries=$(cat ${tmpfile} | wc -l) - # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE - # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512 - # cfg/job.cfg contains (only the attributes to be used are shown below): - # [repository] - # repo_version = 2023.06 - # [architecture] - # os_type = linux - # software_subdir = x86_64/intel/skylake_avx512 - repo_version=$(cfg_get_value "repository" "repo_version") - os_type=$(cfg_get_value "architecture" "os_type") - software_subdir=$(cfg_get_value "architecture" "software_subdir") - accelerator=$(cfg_get_value "architecture" "accelerator") - prefix="${repo_version}/software/${os_type}/${software_subdir}" - - # if we build for an accelerator, the prefix is different - if [[ ! -z ${accelerator} ]]; then - prefix="${prefix}/accel/${accelerator}" - fi - - # extract directories/entries from tarball content - modules_entries=$(grep "${prefix}/modules" ${tmpfile}) - software_entries=$(grep "${prefix}/software" ${tmpfile}) - other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") - other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) - modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) - software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) - - artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" +if [[ $USE_CHECK_BUILD_ARTEFACTS_SCRIPT -eq 0 ]]; then + # first construct comment_artefacts_list + # then use it to set comment_artefacts comment_artifacts_list="" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" - comment_artifacts_list="${comment_artifacts_list}
"
-    if [[ ! -z ${modules} ]]; then
-        while IFS= read -r mod ; do
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
-        done <<< "${modules}"
-    else
-        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
-    fi
-    comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" - comment_artifacts_list="${comment_artifacts_list}
"
-    if [[ ! -z ${software_pkgs} ]]; then
-        while IFS= read -r sw_pkg ; do
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
-        done <<< "${software_pkgs}"
-    else
-        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
-    fi
-    comment_artifacts_list="${comment_artifacts_list}
" - comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})" - comment_artifacts_list="${comment_artifacts_list}
"
-    if [[ ! -z ${other_shortened} ]]; then
-        while IFS= read -r other ; do
-            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
-        done <<< "${other_shortened}"
+
+    # TARBALL should only contain a single tarball
+    if [[ ! -z ${TARBALL} ]]; then
+        # Example of the detailed information for a tarball. The actual result MUST be a
+        # single line (no '\n') or it would break the structure of the markdown table
+        # that holds status updates of a bot job.
+        # 
+        # 

+        #   

+        #     eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz
+        #     size: 234 MiB (245366784 bytes)

+        #     entries: 1234

+        #     modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_

+        #     
+        #       GCC/9.3.0.lua

+        #       GCC/10.3.0.lua

+        #       OpenSSL/1.1.lua
+        #     

+        #     software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_
+        #     
+        #       GCC/9.3.0/

+        #       CMake/3.20.1-GCCcore-10.3.0/

+        #       OpenMPI/4.1.1-GCC-10.3.0/
+        #     

+        #     other under _2023.06/software/linux/x86_64/intel/cascadelake/_
+        #     
+        #       .lmod/cache/spiderT.lua

+        #       .lmod/cache/spiderT.luac_5.1

+        #       .lmod/cache/timestamp
+        #     

+        #   

+        # 

+        size="$(stat --dereference --printf=%s ${TARBALL})"
+        size_mib=$((${size} >> 20))
+        tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX)
+        tar tf ${TARBALL} > ${tmpfile}
+        entries=$(cat ${tmpfile} | wc -l)
+        # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE
+        # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512
+        # cfg/job.cfg contains (only the attributes to be used are shown below):
+        # [repository]
+        # repo_version = 2023.06
+        # [architecture]
+        # os_type = linux
+        # software_subdir = x86_64/intel/skylake_avx512
+        repo_version=$(cfg_get_value "repository" "repo_version")
+        os_type=$(cfg_get_value "architecture" "os_type")
+        software_subdir=$(cfg_get_value "architecture" "software_subdir")
+        accelerator=$(cfg_get_value "architecture" "accelerator")
+        prefix="${repo_version}/software/${os_type}/${software_subdir}"
+
+        # if we build for an accelerator, the prefix is different
+        if [[ ! -z ${accelerator} ]]; then
+          prefix="${prefix}/accel/${accelerator}"
+        fi
+
+        # extract directories/entries from tarball content
+        modules_entries=$(grep "${prefix}/modules" ${tmpfile})
+        software_entries=$(grep "${prefix}/software" ${tmpfile})
+        other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software")
+        other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u)
+        modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u)
+        software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
+
+        artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})"
+        comment_artifacts_list=""
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
+        comment_artifacts_list="${comment_artifacts_list}
"
+        if [[ ! -z ${modules} ]]; then
+            while IFS= read -r mod ; do
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${mod})"
+            done <<< "${modules}"
+        else
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
+        fi
+        comment_artifacts_list="${comment_artifacts_list}
"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
+        comment_artifacts_list="${comment_artifacts_list}
"
+        if [[ ! -z ${software_pkgs} ]]; then
+            while IFS= read -r sw_pkg ; do
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${sw_pkg})"
+            done <<< "${software_pkgs}"
+        else
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+        fi
+        comment_artifacts_list="${comment_artifacts_list}
"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
+        comment_artifacts_list="${comment_artifacts_list}
"
+        if [[ ! -z ${other_shortened} ]]; then
+            while IFS= read -r other ; do
+                comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' ${other})"
+            done <<< "${other_shortened}"
+        else
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
+        fi
+        comment_artifacts_list="${comment_artifacts_list}
"
     else
-        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')"
     fi
-    comment_artifacts_list="${comment_artifacts_list}
" -else - comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')" -fi -comment_artefact_details_fmt="
__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
" -comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" -comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}" + comment_artefact_details_fmt="
__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
" + comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" + comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}" -comment_artefacts_fmt="
_Artefacts_
__ARTEFACTS_LIST__
" -comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" -comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + comment_artefacts_fmt="
_Artefacts_
__ARTEFACTS_LIST__
" + comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" + comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} -echo "${comment_description}" >> ${job_result_file} + echo "${comment_description}" >> ${job_result_file} -# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts -# - this should make use of subsequent steps such as deploying a tarball more -# efficient -echo "status = ${status}" >> ${job_result_file} -echo "artefacts = " >> ${job_result_file} -echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} + # add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts + # - this should make use of subsequent steps such as deploying a tarball more + # efficient + echo "status = ${status}" >> ${job_result_file} + echo "artefacts = " >> ${job_result_file} + echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} -# remove tmpfile -if [[ -f ${tmpfile} ]]; then - rm ${tmpfile} + # remove tmpfile + if [[ -f ${tmpfile} ]]; then + rm ${tmpfile} + fi + +elif [[ -f "$TOPDIR/check-build-artefacts.sh" ]]; then + source "$TOPDIR/check-build-artefacts.sh" +else + echo "ERROR: Required script $TOPDIR/check-build-artefacts.sh not found!" >&2 + exit 1 fi # exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1) diff --git a/bot/test.sh b/bot/test.sh index 4205ea8564..b615be0020 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -135,7 +135,8 @@ mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then - for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + IFS=',' read -r -a modules <<< "$(echo "${LOAD_MODULES}")" + for mod in "${modules[@]}"; do echo "bot/test.sh: loading module '${mod}'" module load ${mod} @@ -172,6 +173,10 @@ EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/test.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" +# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} +export EESSI_ACCELERATOR_TARGET=$(cfg_get_value "architecture" "accelerator") +echo "bot/test.sh: EESSI_ACCELERATOR_TARGET='${EESSI_ACCELERATOR_TARGET}'" + # get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} @@ -187,6 +192,10 @@ COMMON_ARGS+=("--mode" "run") [[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") [[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") +# pass through '--contain' to avoid leaking in scripts into the container session +# note, --pass-through can be used multiple times if needed +COMMON_ARGS+=("--pass-through" "--contain") + # make sure to use the same parent dir for storing tarballs of tmp PREVIOUS_TMP_DIR=${PWD}/previous_tmp @@ -210,8 +219,19 @@ TEST_STEP_ARGS+=("--extra-bind-paths" "/sys/fs/cgroup:/hostsys/fs/cgroup:ro") # add options required to handle NVIDIA support if command_exists "nvidia-smi"; then - echo "Command 'nvidia-smi' found, using available GPU" - TEST_STEP_ARGS+=("--nvidia" "run") + # Accept that this may fail + set +e + nvidia-smi --version + ec=$? + set -e + if [ ${ec} -eq 0 ]; then + echo "Command 'nvidia-smi' found, using available GPU" + TEST_STEP_ARGS+=("--nvidia" "run") + else + echo "Warning: command 'nvidia-smi' found, but 'nvidia-smi --version' did not run succesfully." + echo "This script now assumes this is NOT a GPU node." + echo "If, and only if, the current node actually does contain Nvidia GPUs, this should be considered an error." + fi fi # prepare arguments to test_suite.sh (specific to test step) @@ -219,6 +239,9 @@ declare -a TEST_SUITE_ARGS=() if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then TEST_SUITE_ARGS+=("--generic") fi +if [[ ${SHARED_FS_PATH} ]]; then + TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") +fi # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}") # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}") diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index e959572ab1..6acef9bffa 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -8,10 +8,13 @@ DOT_LMOD = '.lmod' -hook_txt = """require("strict") +hook_prologue = """require("strict") local hook = require("Hook") local open = io.open +""" + +hook_txt = """ local function read_file(path) local file = open(path, "rb") -- r read mode and b binary mode if not file then return nil end @@ -107,36 +110,42 @@ end -local function eessi_cuda_enabled_load_hook(t) +local function eessi_cuda_and_libraries_enabled_load_hook(t) local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. - -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse - -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI + local packagesList = { ["CUDA"] = true, ["cuDNN"] = true } + -- If we try to load any of the modules in packagesList, we check if the + -- full package was installed on the host in host_injections. + -- This is required for end users to build additional software that depends + -- on the package. If the full SDK isn't present, refuse + -- to load the module and print an informative message on how to set up GPU support for EESSI local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/site_specific_config/gpu/.\\n" - if simpleName == 'CUDA' then + if packagesList[simpleName] then + -- simpleName is a module in packagesList -- get the full host_injections path local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections') - -- build final path where the CUDA software should be installed - local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" - local cudaDirExists = isDir(cudaEasyBuildDir) - if not cudaDirExists then + + -- build final path where the software should be installed + local packageEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" + local packageDirExists = isDir(packageEasyBuildDir) + if not packageDirExists then local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI " + advice = advice .. "due to licencing. You will need to install a full copy of the " .. simpleName .. " package where EESSI " advice = advice .. "can find it.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) end end - -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, + -- when loading CUDA (and cu*) enabled modules check if the necessary driver libraries are accessible to the EESSI linker, -- otherwise, refuse to load the requested module and print error message local checkGpu = mt:haveProperty(simpleName,"arch","gpu") local overrideGpuCheck = os.getenv("EESSI_OVERRIDE_GPU_CHECK") if checkGpu and (overrideGpuCheck == nil) then local arch = os.getenv("EESSI_CPU_FAMILY") or "" - local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" - local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cvmfs_repo = os.getenv("EESSI_CVMFS_REPO") or "" + local cudaVersionFile = cvmfs_repo .. "/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = cvmfs_repo .. "/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" local cudaDriverExists = isFile(cudaDriverFile) local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then @@ -190,24 +199,148 @@ end end +local function eessi_scipy_2022b_test_failures_message(t) + local cpuArch = os.getenv("EESSI_SOFTWARE_SUBDIR") + local graceArch = 'aarch64/nvidia/grace' + local fullModuleName = 'SciPy-bundle/2023.02-gfbf-2022b' + local moduleVersionArchMatch = t.modFullName == fullModuleName and cpuArch == graceArch + if moduleVersionArchMatch and not os.getenv("EESSI_IGNORE_MODULE_WARNINGS") then + -- Print a message on loading SciPy-bundle version == 2023.02 informing about the higher number of + -- test failures and recommend using other versions available via EESSI. + -- A message and not a warning as the exit code would break CI runs otherwise. + local simpleName = string.match(t.modFullName, "(.-)/") + local advice = 'The module ' .. t.modFullName .. ' will be loaded. However, note that\\n' + advice = advice .. 'during its building for the CPU microarchitecture ' .. graceArch .. ' from a\\n' + advice = advice .. 'total of 52.730 unit tests a larger number (46) than usually (2-4) failed. If\\n' + advice = advice .. 'you encounter issues while using ' .. t.modFullName .. ', please,\\n' + advice = advice .. 'consider using one of the other versions of ' .. simpleName .. ' that are also provided\\n' + advice = advice .. 'for the same CPU microarchitecture.\\n' + LmodMessage("\\n", advice) + end +end + -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) eessi_espresso_deprecated_message(t) - -- Only apply CUDA hooks if the loaded module is in the EESSI prefix - -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack + eessi_scipy_2022b_test_failures_message(t) + -- Only apply CUDA and cu*-library hooks if the loaded module is in the EESSI prefix + -- This avoids getting an Lmod Error when trying to load a CUDA or cu* module from a local software stack if from_eessi_prefix(t) then - eessi_cuda_enabled_load_hook(t) + eessi_cuda_and_libraries_enabled_load_hook(t) + end +end + +local function using_eessi_accel_stack () + local modulepath = os.getenv("MODULEPATH") or "" + local accel_stack_in_modulepath = false + + -- Check if we are using an EESSI version 2023 accelerator stack by checking if the $MODULEPATH contains + -- a path that starts with /cvmfs/software.eessi.io and contains accel/nvidia/ccNN + for path in string.gmatch(modulepath, '(.-):') do + if string.sub(path, 1, 41) == "/cvmfs/software.eessi.io/versions/2023.06" then + if string.find(path, "accel/nvidia/cc%d%d") then + accel_stack_in_modulepath = true + break + end + end end + return accel_stack_in_modulepath end +local function eessi_removed_module_warning_startup_hook(usrCmd) + if usrCmd == 'load' and not os.getenv("EESSI_SKIP_REMOVED_MODULES_CHECK") then + local CUDA_RELOCATION_MSG = [[All CUDA installations and modules depending on CUDA have been relocated to GPU-specific stacks. + Please see https://www.eessi.io/docs/site_specific_config/gpu/ for more information.]] + + local RELOCATED_CUDA_MODULES = { + ['NCCL'] = CUDA_RELOCATION_MSG, + ['NCCL/2.18.3-GCCcore-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, + ['UCX-CUDA'] = CUDA_RELOCATION_MSG, + ['UCX-CUDA/1.14.1-GCCcore-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, + -- we also have non-CUDA versions of OSU Micro Benchmarks, so only match the CUDA version + ['OSU-Micro-Benchmarks/7.2-gompi-2023a-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, + ['UCC-CUDA'] = CUDA_RELOCATION_MSG, + ['UCC-CUDA/1.2.0-GCCcore-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, + ['CUDA'] = CUDA_RELOCATION_MSG, + ['CUDA/12.1.1'] = CUDA_RELOCATION_MSG, + ['CUDA-Samples'] = CUDA_RELOCATION_MSG, + ['CUDA-Samples/12.1-GCC-12.3.0-CUDA-12.1.1'] = CUDA_RELOCATION_MSG, + } + + local REMOVED_MODULES = { + ['ipympl/0.9.3-foss-2023a'] = 'This module has been replaced by ipympl/0.9.3-gfbf-2023a', + } + + local masterTbl = masterTbl() + local error_msg = "" + -- The CUDA messages should only be shown if the accelerator stack is NOT being used + if not using_eessi_accel_stack() then + for _, module in pairs(masterTbl.pargs) do + if RELOCATED_CUDA_MODULES[module] ~= nil then + error_msg = error_msg .. module .. ': ' .. RELOCATED_CUDA_MODULES[module] .. '\\n\\n' + end + end + end + for _, module in pairs(masterTbl.pargs) do + if REMOVED_MODULES[module] ~= nil then + error_msg = error_msg .. module .. ': ' .. REMOVED_MODULES[module] .. '\\n\\n' + end + end + if error_msg ~= "" then + LmodError('\\n' .. error_msg .. 'If you know what you are doing and you want to ignore this check for removed/relocated modules, set $EESSI_SKIP_REMOVED_MODULES_CHECK to any value.') + end + end +end + +function eessi_startup_hook(usrCmd) + eessi_removed_module_warning_startup_hook(usrCmd) +end + +hook.register("startup", eessi_startup_hook) hook.register("load", eessi_load_hook) +""" + +hook_epilogue = """ -- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site load_site_specific_hooks() """ +# This hook is only for zen4. +hook_txt_zen4 = """ +local function hide_2022b_modules(modT) + -- modT is a table with: fullName, sn, fn and isVisible + -- The latter is a boolean to determine if a module is visible or not + + local tcver = modT.fullName:match("gfbf%-(20[0-9][0-9][ab])") or + modT.fullName:match("gompi%-(20[0-9][0-9][ab])") or + modT.fullName:match("foss%-(20[0-9][0-9][ab])") or + modT.fullName:match("GCC%-([0-9]*.[0-9]*.[0-9]*)") or + modT.fullName:match("GCCcore%-([0-9]*.[0-9]*.[0-9]*)") + + -- if nothing matches, return + if tcver == nil then return end + + -- if we have matches, check if the toolchain version is either 2022b or 12.2.0 + if parseVersion(tcver) == parseVersion("2022b") or parseVersion(tcver) == parseVersion("12.2.0") then + modT.isVisible = false + end +end + +hook.register("isVisibleHook", hide_2022b_modules) +""" + +# Append conditionally for zen4 +eessi_software_subdir_override = os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") +if eessi_software_subdir_override == "x86_64/amd/zen4": + hook_txt = hook_txt + hook_txt_zen4 + +# Concatenate hook prologue, body and epilogue +# Note that this has to happen after any conditional items have been added to the hook_txt +hook_txt = hook_prologue + hook_txt + hook_epilogue + def error(msg): sys.stderr.write("ERROR: %s\n" % msg) sys.exit(1) diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-001-system.yml new file mode 100644 index 0000000000..47a05b5f30 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.4-001-system.yml @@ -0,0 +1,20 @@ +easyconfigs: + - EasyBuild-4.8.2.eb + - EasyBuild-4.9.0.eb + - EasyBuild-4.9.1.eb + - EasyBuild-4.9.2.eb + - EasyBuild-4.9.3.eb + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 + - Nextflow-23.10.0.eb + - ReFrame-4.3.3.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22183 + from-commit: 2b2fe53c885799cbf13b77ddfa9532c48b296e9d + - ReFrame-4.6.2.eb + - Pandoc-3.6.2.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22493 + from-commit: da8ed20bad0dd1c6533c568f6c4fbb7c3d15342e diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250215-eb-4.9.4-CUDA-update-module-files.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250215-eb-4.9.4-CUDA-update-module-files.yml new file mode 100644 index 0000000000..d289ab1575 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/rebuilds/20250215-eb-4.9.4-CUDA-update-module-files.yml @@ -0,0 +1,14 @@ +# 2025.02.15 +# We need to update the module files for all CUDA installations to add +# additional directories to LIBRARY_PATH. +# See https://github.com/easybuilders/easybuild-easyblocks/pull/3516 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA + force: True + include-easyblocks-from-commit: 3469151ce7e4f85415c877dee555aeea7691c757 + - CUDA-12.4.0.eb: + options: + accept-eula-for: CUDA + include-easyblocks-from-commit: 3469151ce7e4f85415c877dee555aeea7691c757 diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml new file mode 100644 index 0000000000..60d82d46ad --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml @@ -0,0 +1,4 @@ +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml new file mode 100644 index 0000000000..8a55c5aa7e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/zen4_h100/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -0,0 +1,4 @@ +easyconfigs: +- CUDA-12.4.0.eb: + options: + accept-eula-for: CUDA diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 3f6590c3cd..64dc9d1e8c 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -73,3 +73,5 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20238 from-pr: 20238 + - Z3-4.12.2-GCCcore-12.3.0-Python-3.11.3.eb + - GDRCopy-2.3.1-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-001-system.yml new file mode 100644 index 0000000000..362e42ef93 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-001-system.yml @@ -0,0 +1,9 @@ +easyconfigs: + - Pandoc-3.6.2.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22493 + from-commit: da8ed20bad0dd1c6533c568f6c4fbb7c3d15342e + - EasyBuild-5.0.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22606 + from-commit: 655adeb30b1b7a729650fd1fcc8660b25fb5f631 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 19d685b109..537c7705d7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -45,3 +45,17 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22145 from-commit: 31478e5c9869de3add74d0a02dd5df01ea65b21e + - archspec-0.2.5-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22235 + from-commit: 01dd97ea62fe4d7d0df040ede3af03eb2f1b8641 + - OpenCV-4.8.1-foss-2023a-contrib.eb + - FALL3D-9.0.1-gompi-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22610 + from-commit: 10e47e2fdd4f23fd5a52ae167771f8e9c78da411 + - lit-18.1.2-GCCcore-12.3.0.eb + # needed due to changed/new dependencies for R-bundle-CRAN-2023.12-foss-2023a.eb + # without any from-commit this will use a version from April 6, 2024 + # https://github.com/easybuilders/easybuild-easyconfigs/blob/88f6f9c7439c535e62461e6e71b1961c7be118b8/easybuild/easyconfigs/l/lit/lit-18.1.2-GCCcore-12.3.0.eb + # EB 5.0.0 has a few changes to that diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml index ee161b8b9c..18620a57e0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023b.yml @@ -25,3 +25,23 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa + - DP3-6.2-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - WSClean-3.5-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - EveryBeam-0.6.1-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - lit-18.1.7-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20902 + from-commit: 79f4cf21490f7f5b187af889be5426c1332a497d + - astropy-7.0.0-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22469 + from-commit: fc22841fef99cbb2a221c18029b15e692e78c27c diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml new file mode 100644 index 0000000000..e08f6a8773 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-5.0.0-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - Siesta-5.2.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22292 + # and https://github.com/easybuilders/easybuild-easyconfigs/pull/22646 + from-commit: b212c00fdc3983678037429719f1b210cb978b42 diff --git a/easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml new file mode 100644 index 0000000000..e25c418d62 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023a-CUDA.yml @@ -0,0 +1,20 @@ +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3516 + include-easyblocks-from-commit: 3469151ce7e4f85415c877dee555aeea7691c757 + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb + - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb + - UCC-CUDA-1.2.0-GCCcore-12.3.0-CUDA-12.1.1.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c + - cuDNN-8.9.2.26-CUDA-12.1.1.eb + - LightGBM-4.5.0-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 + from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 + - NCCL-2.18.3-GCCcore-12.3.0-CUDA-12.1.1.eb diff --git a/easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml b/easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml new file mode 100644 index 0000000000..46d93587af --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/accel/nvidia/eessi-2023.06-eb-4.9.4-2023b-CUDA.yml @@ -0,0 +1,25 @@ +easyconfigs: + - CUDA-12.4.0.eb: + options: + accept-eula-for: CUDA + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3516 + include-easyblocks-from-commit: 3469151ce7e4f85415c877dee555aeea7691c757 + - UCX-CUDA-1.15.0-GCCcore-13.2.0-CUDA-12.4.0.eb + - UCC-CUDA-1.2.0-GCCcore-13.2.0-CUDA-12.4.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21565 + from-commit: 46141a3f40e699433fac03af2d3ed81bd5a62da7 + - OSU-Micro-Benchmarks-7.5-gompi-2023b-CUDA-12.4.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21926 + # from-commit: de79ec74eb076e1aceda5e21235a73c05ed6764c + # use merge commit + from-commit: f3bd10b19f7caf4de3302bc7a73749341db9c7d8 + - GROMACS-2024.4-foss-2023b-CUDA-12.4.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21549 + # commit below is the merge commit + from-commit: 12b53341343967ce5a402fe8190a3c85bce7d49b + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3523 + # commit below is last one before the merge commit + include-easyblocks-from-commit: 90495ed23d26b3d5fd8162bf5d7b4c073a0682fe diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 100644 index 0000000000..d01c1f0639 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1,11 @@ +easyconfigs: +# from here on built originally with EB 4.8.2 +# need to keep building Qt5 with 4.8.2 because more recent versions include an +# updated easyblock for python which doesn't work correctly for Python-2.7.18 + - Qt5-5.15.7-GCCcore-12.2.0.eb +# try to also build QE with EB 4.8.2 --> worked but it looks like we may not have +# used the FoX library, see https://github.com/easybuilders/easybuild-easyconfigs/pull/20070/files#diff-b5e488fd7d21901d2307ed7994868853061fb5c60080e91cb186e2f937e45522 +# so we move building QE 7.2 with EB 4.9.4; that failed plus QE was not built +# with the FoX library for any other of the supported CPU microarchitectures, +# hence we keep building it with EB 4.8.2 (here in this easystack file) + - QuantumESPRESSO-7.2-foss-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..f1e0b7f971 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: +# in the latest easyconfig, Time::HiRes extension has been removed +# we need to build Perl-bundle-CPAN using EB 4.9.2 to stay in sync with builds for other archs in EESSI +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/21198 +# Perl-bundle-CPAN is a dependency of BioPerl + - Perl-bundle-CPAN-5.36.1-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-001-system.yml new file mode 100644 index 0000000000..47a05b5f30 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-001-system.yml @@ -0,0 +1,20 @@ +easyconfigs: + - EasyBuild-4.8.2.eb + - EasyBuild-4.9.0.eb + - EasyBuild-4.9.1.eb + - EasyBuild-4.9.2.eb + - EasyBuild-4.9.3.eb + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 + - Nextflow-23.10.0.eb + - ReFrame-4.3.3.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22183 + from-commit: 2b2fe53c885799cbf13b77ddfa9532c48b296e9d + - ReFrame-4.6.2.eb + - Pandoc-3.6.2.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22493 + from-commit: da8ed20bad0dd1c6533c568f6c4fbb7c3d15342e diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2022b.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2022b.yml new file mode 100644 index 0000000000..1302be948f --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2022b.yml @@ -0,0 +1,156 @@ +easyconfigs: +# from here on built originally with EB 4.8.2 +# make sure Python-3.10.8-GCCcore-12.2.0-bare.eb is built from correct PR/commit +# commit 1ee17c0f7726c69e97442f53c65c5f041d65c94f from +# https://github.com/easybuilders/easybuild-easyblocks/pull/3352 was included +# since EB 4.9.3 --> no special treating needed +# same applies to Python-3.10.8-GCCcore-12.2.0 +# +# originally built with EB 4.8.2, PR 19159 was included since EB 4.9.0, PR 3492 +# was included in EB 5.0.0 -> need to keep commit for easyblock +# - OpenBLAS-0.3.21-GCC-12.2.0.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19159 +# # required for Sapphire Rapids support +# from-pr: 19159 +# # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 +# include-easyblocks-from-pr: 3492 + - OpenBLAS-0.3.21-GCC-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + include-easyblocks-from-commit: 4cef6cea5badad0846be3f536d2af70433ff8c51 +# originally built with EB 4.8.2, PR 19940 was included since EB 4.9.1 +# - OpenMPI-4.1.4-GCC-12.2.0.eb: +# options: +# from-pr: 19940 + - OpenMPI-4.1.4-GCC-12.2.0.eb + - foss-2022b.eb +# originally built with EB 4.8.2, PR 19339 was included since EB 4.9.0 +# - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: +# options: +# from-pr: 19339 + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb +# moved Qt5 to easystack file used with EB 4.8.2 because it needs an older +# version of the python easyblock +# - Qt5-5.15.7-GCCcore-12.2.0.eb +# we try to build QE with EB 4.8.2 too, that worked; building with updated +# easyconfigs (via EB 4.9.4) didn't work, and wasn't done for any of the other +# supported CPU microarchitectures. Hence, we build QE with 4.8.2 +# - QuantumESPRESSO-7.2-foss-2022b.eb +# +# from here on built originally with EB 4.9.0 +# includes dependencies Boost/1.81.0 and Boost.MPI/1.81.0 for which we have to +# use updated easyconfigs (via from-commit) because the download URLs have +# changed +# +# originally built with EB 4.9.0, PR 20298 was included since 4.9.2 and no more +# updates to it since then +# - Highway-1.0.3-GCCcore-12.2.0.eb: +# options: +# from-pr: 20298 + - Highway-1.0.3-GCCcore-12.2.0.eb + - SciPy-bundle-2023.02-gfbf-2022b.eb +# Boost-1.81.0-GCC-12.2.0.eb is a dependency for GDAL and the sources URL for Boost +# has recently changed by PR 22157 +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/22157 + - Boost-1.81.0-GCC-12.2.0.eb: + options: + from-commit: 5bebccf792ccf35a8ee3250bc8fed86dff5d5df9 + - GDAL-3.6.2-foss-2022b.eb +# originally built with EB 4.9.0, PR 19324 was included since 4.9.1; there were a +# few additional changes to the easyconfig, hence we use the merge commit for 19324 +# - waLBerla-6.1-foss-2022b.eb: +# options: +# from-pr: 19324 +# Boost.MPI-1.81.0-gompi-2022b.eb is a dependency for waLBerla and the sources URL for +# Boost.MPI has recently changed by PR 22240 +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/22240 + - Boost.MPI-1.81.0-gompi-2022b.eb: + options: + from-commit: e610fe1ac5393d1de668a466fdaaea74c580ee03 + - waLBerla-6.1-foss-2022b.eb: + options: + from-commit: 11daa230014b22387c28712d2ce93c45618058f6 + - WRF-4.4.1-foss-2022b-dmpar.eb +# originally built with EB 4.9.0, PR 20086 was included since 4.9.1; there were a +# few additional changes to the easyconfig, hence we use the merge commit for 20086 +# - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb: +# options: +# from-pr: 20086 + - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb: + options: + from-commit: a0eff4515ecad4fe37d7c018c95526ad4a777de7 +# originally built with EB 4.9.0, PR 20238 was included since 4.9.1; there were +# additional changes, particularly addressing CVE 2024-27322 that was included +# since EB 4.9.2, hence we simply use the easyconfig available with EB 4.9.4 +# - R-4.2.2-foss-2022b.eb: +# options: +# from-pr: 20238 + - R-4.2.2-foss-2022b.eb +# from here on built originally with EB 4.9.1 +# originally built with EB 4.9.1, PR 20379 was included since 4.9.2; no more +# updates to the easyconfig since then +# - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20379 +# from-pr: 20379 + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb + - ParaView-5.11.1-foss-2022b.eb + - ASE-3.22.1-gfbf-2022b.eb + - SEPP-4.5.1-foss-2022b.eb + - Valgrind-3.21.0-gompi-2022b.eb +# from here on built originally with EB 4.9.2 + - BLAST+-2.14.0-gompi-2022b.eb +# originally built with EB 4.9.2; PR 21136 included since EB 4.9.3 +# - BioPerl-1.7.8-GCCcore-12.2.0.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 +# from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - BioPerl-1.7.8-GCCcore-12.2.0.eb + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - h5py-3.8.0-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb +# - MUMPS-5.6.1-foss-2022b-metis.eb +# - GL2PS-1.4.2-GCCcore-12.2.0.eb +# - GST-plugins-base-1.22.1-GCC-12.2.0.eb +# - wxWidgets-3.2.2.1-GCC-12.2.0.eb +# - Archive-Zip-1.68-GCCcore-12.2.0.eb +# - jemalloc-5.3.0-GCCcore-12.2.0.eb +# - Judy-1.0.5-GCCcore-12.2.0.eb +# - libaio-0.3.113-GCCcore-12.2.0.eb +# - Z3-4.12.2-GCCcore-12.2.0.eb +# - tbb-2021.10.0-GCCcore-12.2.0.eb +# - dask-2023.7.1-foss-2022b.eb +# - netcdf4-python-1.6.3-foss-2022b.eb +# - Ruby-3.2.2-GCCcore-12.2.0.eb +## originally built with EB 4.9.2; PR 21526 and PR 3467 included since EB 5.0.0, thus need to use *from-commit +# - ROOT-6.26.10-foss-2022b.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 +# from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df +# # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 +# include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..a5ecb41eb9 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,210 @@ +easyconfigs: + - GCCcore-12.3.0.eb + - OpenBLAS-0.3.23-GCC-12.3.0.eb + - Rust-1.70.0-GCCcore-12.3.0.eb + - OpenMPI-4.1.5-GCC-12.3.0 + - foss-2023a.eb + - pybind11-2.11.1-GCCcore-12.3.0.eb + - SciPy-bundle-2023.07-gfbf-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21693 + from-commit: 7c5144d2c1a061cd9f08b5901970b7f6ec5eb5c0 + - TensorFlow-2.13.0-foss-2023a.eb + - X11-20230603-GCCcore-12.3.0.eb + - HarfBuzz-5.3.1-GCCcore-12.3.0.eb + - Qt5-5.15.10-GCCcore-12.3.0.eb + - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb + - LHAPDF-6.5.4-GCC-12.3.0.eb + - LoopTools-2.15-GCC-12.3.0.eb + - R-4.3.2-gfbf-2023a.eb + - Boost-1.82.0-GCC-12.3.0.eb: + options: + # source URLs for Boost have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22157 + from-commit: 5bebccf792ccf35a8ee3250bc8fed86dff5d5df9 + - netCDF-4.9.2-gompi-2023a.eb + - FFmpeg-6.0-GCCcore-12.3.0.eb + - ALL-0.9.2-foss-2023a.eb + - CDO-2.2.2-gompi-2023a.eb + - BWA-0.7.17-20220923-GCCcore-12.3.0.eb + - METIS-5.1.0-GCCcore-12.3.0.eb + - SCOTCH-7.0.3-gompi-2023a.eb + - CGAL-5.6-GCCcore-12.3.0.eb + - ParaView-5.11.2-foss-2023a.eb + - gnuplot-5.4.8-GCCcore-12.3.0.eb + - Boost.MPI-1.82.0-gompi-2023a.eb: + options: + # source URLs for Boost.MPI have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22240 + # Boost.MPI is a dependency of ESPResSo + from-commit: e610fe1ac5393d1de668a466fdaaea74c580ee03 + - ESPResSo-4.2.1-foss-2023a.eb + - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb + - Pillow-10.0.0-GCCcore-12.3.0.eb + - sympy-1.12-gfbf-2023a.eb + - networkx-3.1-gfbf-2023a.eb + - expecttest-0.1.5-GCCcore-12.3.0.eb + - PyYAML-6.0-GCCcore-12.3.0.eb + - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb + - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb + - pytest-shard-0.1.2-GCCcore-12.3.0.eb + - scikit-learn-1.3.1-gfbf-2023a.eb + - MUMPS-5.6.1-foss-2023a-metis.eb: + options: + # source URLs for MUMPS have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22582 + # MUMPS is a dependency of snakemake + from-commit: 0437ff1ad34283398f55d4a6e01e6540b1ae9688 + - snakemake-8.4.2-foss-2023a.eb + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb + - PyTorch-2.1.2-foss-2023a.eb + - matplotlib-3.7.2-gfbf-2023a.eb + - PyQt5-5.15.10-GCCcore-12.3.0.eb + - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb + - dask-2023.9.2-foss-2023a.eb + - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb + - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb + - Z3-4.12.2-GCCcore-12.3.0.eb + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb + - OpenJPEG-2.5.0-GCCcore-12.3.0.eb + - Highway-1.0.4-GCCcore-12.3.0.eb + - ELPA-2023.05.001-foss-2023a.eb + - libxc-6.2.2-GCC-12.3.0.eb: + options: + # possible change in gitlabs tarball packaging, affected by .gitattributes + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22580 + from-commit: 456d64bbeacf465e8f7e7ff378864e26352d045d + - ParMETIS-4.0.3-gompi-2023a.eb: + options: + # source URLs for ParMETIS-4.0.3 have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22579 + # ParMETIS-4.0.3 is a dependency of SuperLU_DIST-8.1.2 + from-commit: 977e5208a720f23ace41b83b84da8b717d0aeada + - SuperLU_DIST-8.1.2-foss-2023a.eb + - PETSc-3.20.3-foss-2023a.eb + - MODFLOW-6.4.4-foss-2023a.eb + - NLopt-2.7.1-GCCcore-12.3.0.eb + - nettle-3.9.1-GCCcore-12.3.0.eb + - Xvfb-21.1.8-GCCcore-12.3.0.eb + - libsndfile-1.2.2-GCCcore-12.3.0.eb + - PostgreSQL-16.1-GCCcore-12.3.0.eb + - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb + - GDAL-3.7.1-foss-2023a.eb + - ncdu-1.18-GCC-12.3.0.eb + - SAMtools-1.18-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + # (additional extensions have been added) + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21948 + from-commit: f9cfe6ac7d9019970c2be3e8b09db4d846cf005a + - ipympl-0.9.3-gfbf-2023a.eb + - ESPResSo-4.2.2-foss-2023a.eb + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - WhatsHap-2.2-foss-2023a.eb + - BLAST+-2.14.1-gompi-2023a.eb + - Valgrind-3.21.0-gompi-2023a.eb + - OrthoFinder-2.5.5-foss-2023a.eb + - OpenFOAM-10-foss-2023a.eb + - OpenFOAM-11-foss-2023a.eb + - BCFtools-1.18-GCC-12.3.0.eb + - BWA-0.7.18-GCCcore-12.3.0.eb + - CapnProto-1.0.1-GCCcore-12.3.0.eb + - DendroPy-4.6.1-GCCcore-12.3.0.eb + - DIAMOND-2.1.8-GCC-12.3.0.eb + - FastME-2.1.6.3-GCC-12.3.0.eb + - fastp-0.23.4-GCC-12.3.0.eb + - HMMER-3.4-gompi-2023a.eb + - IQ-TREE-2.3.5-gompi-2023a.eb + - KronaTools-2.8.1-GCCcore-12.3.0.eb + - LSD2-2.4.1-GCCcore-12.3.0.eb + - MAFFT-7.520-GCC-12.3.0-with-extensions.eb + - ncbi-vdb-3.0.10-gompi-2023a.eb + - MetalWalls-21.06.1-foss-2023a.eb + - QuantumESPRESSO-7.3.1-foss-2023a.eb + - CP2K-2023.1-foss-2023a.eb + - amdahl-0.3.1-gompi-2023a.eb + - LLVM-14.0.6-GCCcore-12.3.0-llvmlite.eb + - numba-0.58.1-foss-2023a.eb + - librosa-0.10.1-foss-2023a.eb + - xarray-2023.9.0-gfbf-2023a.eb + - SciTools-Iris-3.9.0-foss-2023a.eb + - OpenFOAM-v2312-foss-2023a.eb + - BioPerl-1.7.8-GCCcore-12.3.0.eb + - grpcio-1.57.0-GCCcore-12.3.0.eb + - orjson-3.9.15-GCCcore-12.3.0.eb + - wradlib-2.0.3-foss-2023a.eb + - MBX-1.1.0-foss-2023a.eb + - Transrate-1.0.3-GCC-12.3.0.eb + - Critic2-1.2-foss-2023a.eb + - LRBinner-0.1-foss-2023a.eb + - Redland-1.0.17-GCC-12.3.0.eb + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb + - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb + - R-bundle-CRAN-2023.12-foss-2023a.eb + - archspec-0.2.1-GCCcore-12.3.0.eb + - ROOT-6.30.06-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 + - waLBerla-6.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 + from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 + - mpl-ascii-0.10.0-gfbf-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21679 + from-commit: 7106f63160b1418d605882dd02ba151d099300bd + - jedi-0.19.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650 + from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13 + - Solids4foam-2.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21606 + from-commit: 63562c58acf1be64407192b6862c3bd80253d2e0 + - Cassiopeia-2.0.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 + from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd + - LightGBM-4.5.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 + from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 + - OpenFOAM-v2406-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3519 + include-easyblocks-from-commit: e4a3ff1932350d575dffc7597435609fad6dd691 + - Paraver-4.11.4-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20230 + from-commit: 91c8df6b4c0810061e9f325427c9c79e961bc4b0 + - Tombo-1.5.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 + from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 + - elfx86exts-0.6.2-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22145 + from-commit: 31478e5c9869de3add74d0a02dd5df01ea65b21e + - archspec-0.2.5-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22235 + from-commit: 01dd97ea62fe4d7d0df040ede3af03eb2f1b8641 + - GDRCopy-2.3.1-GCCcore-12.3.0.eb + - OpenCV-4.8.1-foss-2023a-contrib.eb + - FALL3D-9.0.1-gompi-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22610 + from-commit: 10e47e2fdd4f23fd5a52ae167771f8e9c78da411 + - lit-18.1.2-GCCcore-12.3.0.eb + # needed due to changed/new dependencies for R-bundle-CRAN-2023.12-foss-2023a.eb + # without any from-commit this will use a version from April 6, 2024 + # https://github.com/easybuilders/easybuild-easyconfigs/blob/88f6f9c7439c535e62461e6e71b1961c7be118b8/easybuild/easyconfigs/l/lit/lit-18.1.2-GCCcore-12.3.0.eb + # EB 5.0.0 has a few changes to that diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023b.yml new file mode 100644 index 0000000000..2d48f163d6 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-4.9.4-2023b.yml @@ -0,0 +1,224 @@ +easyconfigs: +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - GCCcore-13.2.0.eb: +# options: +# from-pr: 19974 + - GCCcore-13.2.0.eb + - GCC-13.2.0.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - OpenMPI-4.1.6-GCC-13.2.0: +# options: +# from-pr: 19940 + - OpenMPI-4.1.6-GCC-13.2.0.eb + - foss-2023b.eb + - SciPy-bundle-2023.11-gfbf-2023b.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - netCDF-4.9.2-gompi-2023b.eb: +# options: +# from-pr: 19534 + - netCDF-4.9.2-gompi-2023b.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - matplotlib-3.8.2-gfbf-2023b.eb: +# options: +# from-pr: 19552 + - matplotlib-3.8.2-gfbf-2023b.eb + - Boost-1.83.0-GCC-13.2.0.eb: + options: + # source URLs for Boost have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22157 + # Boost is a dependency of AOFlagger + from-commit: 5bebccf792ccf35a8ee3250bc8fed86dff5d5df9 + - Boost.Python-1.83.0-GCC-13.2.0.eb: + options: + # source URLs for Boost.* have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22240 + # Boost.Python is a dependency of AOFlagger + from-commit: e610fe1ac5393d1de668a466fdaaea74c580ee03 + - wget-1.21.4-GCCcore-13.2.0.eb: + options: + # way to define source for wget has changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22091 + # wget is a dependency of AOFlagger + from-commit: 9487eb335902fae6c184f7ee03711fd6c09b1710 +# originally built with EB 4.9.0, PRs were included since EB 4.9.1 +# - AOFlagger-3.4.0-foss-2023b.eb: +# options: +# from-pr: 19840 +# include-easyblocks-from-pr: 3088 + - AOFlagger-3.4.0-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - arpack-ng-3.9.0-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - Armadillo-12.8.0-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - casacore-3.5.0-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - IDG-1.2.0-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - EveryBeam-0.5.2-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - DP3-6.0-foss-2023b.eb +# originally built with EB 4.9.0, PRs were included since EB 4.9.1, same as AOFlagger + - WSClean-3.4-foss-2023b.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - CDO-2.2.2-gompi-2023b.eb: +# options: +# from-pr: 19792 + - CDO-2.2.2-gompi-2023b.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - python-casacore-3.5.2-foss-2023b.eb: +# options: +# from-pr: 20089 + - python-casacore-3.5.2-foss-2023b.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - libspatialindex-1.9.3-GCCcore-13.2.0.eb: +# options: +# from-pr: 19922 + - libspatialindex-1.9.3-GCCcore-13.2.0.eb + - LittleCMS-2.15-GCCcore-13.2.0.eb + - giflib-5.2.1-GCCcore-13.2.0.eb + - OpenJPEG-2.5.0-GCCcore-13.2.0.eb + - libwebp-1.3.2-GCCcore-13.2.0.eb + - Wayland-1.22.0-GCCcore-13.2.0.eb +# originally built with EB 4.9.0, PR was included since EB 4.9.1 +# - Qt5-5.15.13-GCCcore-13.2.0.eb: +# options: +# from-pr: 20201 + - Qt5-5.15.13-GCCcore-13.2.0.eb + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb +# from here on easyconfigs were originally built with EB 4.9.1 + - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21671 + # we need to use from-commit because the PR was merged after EB 4.9.4 was + # released + from-commit: c38f0637504bcd66e6f7f80277552934e1b03127 +# originally built with EB 4.9.1, PR 20522 was included since EB 4.9.2 +# - GROMACS-2024.1-foss-2023b.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20522 +# from-commit: a0a467a88506c765a93a96b20d7a8fcb01d46b24 + - GROMACS-2024.1-foss-2023b.eb + - NLTK-3.8.1-foss-2023b.eb +# originally built with EB 4.9.1, PR 20792 was included since EB 4.9.2 +# - Valgrind-3.23.0-gompi-2023b.eb: +# options: +# from-pr: 20792 + - Valgrind-3.23.0-gompi-2023b.eb +# from here on easyconfigs were originally built with EB 4.9.2 + - IPython-8.17.2-GCCcore-13.2.0.eb + - dlb-3.4-gompi-2023b.eb +# originally built with EB 4.9.2, PR 20889 was included since EB 4.9.3 +# - pystencils-1.3.4-gfbf-2023b.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 +# from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 + - pystencils-1.3.4-gfbf-2023b.eb +# originally built with EB 4.9.2, PR 21017 was included since EB 4.9.3, PR 3393 +# was included since EB 4.9.3 +# - Extrae-4.2.0-gompi-2023b.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21017 +# from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf +# # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393 +# include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c + - Extrae-4.2.0-gompi-2023b.eb + - Boost.MPI-1.83.0-gompi-2023b.eb: + options: + # source URLs for Boost.* have changed, corresponding PR is + # https://github.com/easybuilders/easybuild-easyconfigs/pull/22240 + # Boost.MPI is a dependency of pyMBE + from-commit: e610fe1ac5393d1de668a466fdaaea74c580ee03 +# originally built with EB 4.9.2, PR 21034 was included since EB 4.9.3 +# - pyMBE-0.8.0-foss-2023b.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 +# from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 + - pyMBE-0.8.0-foss-2023b.eb +# originally built with EB 4.9.2, PR 21200 was included since EB 4.9.3 +# - STAR-2.7.11b-GCC-13.2.0.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 +# from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 + - STAR-2.7.11b-GCC-13.2.0.eb + - HPL-2.3-foss-2023b.eb +# originally built with EB 4.9.2, PR 21366 was included since EB 4.9.3 +# - R-bundle-CRAN-2024.06-foss-2023b.eb: +# options: +# # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 +# # we use a commit from the Brunsli PR here to get rid of the Highway dependency +# from-commit: 1736a123b1685836452587a5c51793257570bb2d + - R-bundle-CRAN-2024.06-foss-2023b.eb +# from here on easyconfigs were originally built with EB 4.9.3 + - GROMACS-2024.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 + from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b +# originally built with EB 4.9.3, PR 21436 was merged into develop before EB +# 4.9.4 was released, but maybe it wasn't included in EB 4.9.4, to be on the safe +# side we just use the commit, PR 3569 was targetting Sapphire Rapids, so likely +# that is not needed here, however we keep it to stay in sync with what was used +# for Sapphire Rapids + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3569 + include-easyblocks-from-commit: 362b4679193612e04abe336fa041e2a34d183991 +# from here on easyconfigs were originally built with EB 4.9.4 + - SIONlib-1.7.7-GCCcore-13.2.0-tools.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21752 + # NOTE, commit is the last one before the merge commit for PR 21752 + from-commit: 6b8b53493a1188a5baa56a133574daac239730e7 + - Score-P-8.4-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 + # NOTE, commit is the last one before the merge commit for PR 3496 + include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb + - Cython-3.0.10-GCCcore-13.2.0.eb + - Mustache-1.3.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 + # NOTE, commit is the last one before the merge commit for PR 21783 + from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 + - GDRCopy-2.4-GCCcore-13.2.0.eb + - GROMACS-2024.4-foss-2023b.eb: + options: + # https://github.com/easybuilders/easybuild-easyconfigs/pull/21851 + # NOTE, below commit is the merge commit for PR 21851 + from-commit: f0fa64b440deaf5fb0a6d26ff1bb3e9f36626c8a + - SlurmViewer-1.0.1-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21899 + # NOTE, below commit is the merge commit for PR 21899 + from-commit: 0bdeb23c9ea5a3caefd353ecd936919424c1bba4 + - wxWidgets-3.2.6-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 + # NOTE, below commit is the merge commit for PR 21915 + from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa + - DP3-6.2-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + # NOTE, below commit is the merge commit for PR 21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - WSClean-3.5-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + # NOTE, below commit is the merge commit for PR 21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - EveryBeam-0.6.1-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + # NOTE, below commit is the merge commit for PR 21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 +# a few apps installed in late march/early april 2025 + - lit-18.1.7-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20902 + from-commit: 79f4cf21490f7f5b187af889be5426c1332a497d + - astropy-7.0.0-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22469 + from-commit: fc22841fef99cbb2a221c18029b15e692e78c27c diff --git a/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-5.0.0-2023a.yml b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-5.0.0-2023a.yml new file mode 100644 index 0000000000..d162b1a22a --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/grace/eessi-2023.06-eb-5.0.0-2023a.yml @@ -0,0 +1,5 @@ +easyconfigs: + - Siesta-5.2.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22731 + from-commit: b5e36fe564794234cf2d79621ad2d61c3980e5a7 diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml new file mode 100644 index 0000000000..5b90d44a89 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20241015-eb-4.9.4-LAMMPS-generic-builds.yml @@ -0,0 +1,16 @@ +# 2024.10.15 +# Generic build of LAMMPS have optimizations for the build host CPU, +# this is fixed in https://github.com/easybuilders/easybuild-easyblocks/pull/3484 +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: + options: + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484 + include-easyblocks-from-commit: 3671c5b7c238c7dc8aadd2c510329770ef1bdcdf + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 + # see: https://github.com/easybuilders/easybuild-easyblocks/pull/3484 + include-easyblocks-from-commit: 3671c5b7c238c7dc8aadd2c510329770ef1bdcdf + + diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250212-eb-4.9.4-R-bundle-CRAN-2023.12-rebuild-to-sync-exts-between-archs.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250212-eb-4.9.4-R-bundle-CRAN-2023.12-rebuild-to-sync-exts-between-archs.yml new file mode 100644 index 0000000000..d60898ce60 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250212-eb-4.9.4-R-bundle-CRAN-2023.12-rebuild-to-sync-exts-between-archs.yml @@ -0,0 +1,6 @@ +# 2025.02.12 +# Rebuild R-bundle-CRAN to synchronise the extensions between archs. +# This is necessary because the Zen4 version was not built with the same EB +# version (and exts had changed) +easyconfigs: + - R-bundle-CRAN-2023.12-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250223-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python-sapphirerapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250223-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python-sapphirerapids.yml new file mode 100644 index 0000000000..3a413eec53 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250223-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python-sapphirerapids.yml @@ -0,0 +1,15 @@ +# 2025-02-23 +# Move setuptools_scm extension from hatchling to Python by rebuilding +# all affected modules with EasyBuild 4.9.1. +# This solves an issue with pyarrow, which is part of the Arrow installation. +# https://github.com/easybuilders/easybuild-easyconfigs/pull/19777 +# https://github.com/easybuilders/easybuild-easyconfigs/issues/19849 +easyconfigs: +# This hatchling version is already rebuilt in 20250223-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphirerapids.yml, +# so we skip it here. +# - hatchling-1.18.0-GCCcore-12.3.0.eb + - hatchling-1.18.0-GCCcore-13.2.0.eb + - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb + - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + - Python-3.11.3-GCCcore-12.3.0.eb + - Python-3.11.5-GCCcore-13.2.0.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250224-eb-4.9.2-Python-ctypes-sapphirerapids.yml similarity index 98% rename from easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20250224-eb-4.9.2-Python-ctypes-sapphirerapids.yml index 1d9acf0988..306611e6e6 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20250119-eb-4.9.2-Python-ctypes-sapphire_rapids.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250224-eb-4.9.2-Python-ctypes-sapphirerapids.yml @@ -1,4 +1,4 @@ -# 2025.01.19 +# 2025.02.24 # Python ctypes relies on LD_LIBRARY_PATH and doesn't respect rpath linking. There is a workaround # for the EasyBuild context in https://github.com/easybuilders/easybuild-easyblocks/pull/3352. # diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphirerapids.yml similarity index 96% rename from easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphirerapids.yml index a86eb166bd..c6c7bc04dd 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20250120-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphire_rapids.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-hatchling-1.18.0-updated-easyconfig-sapphirerapids.yml @@ -1,4 +1,4 @@ -# 2025.01.20 +# 2025.02.28 # hatchling-1.18.0 rebuild to account for easyconfig changed upstream # see https://gitlab.com/eessi/support/-/issues/85 and # https://github.com/easybuilders/easybuild-easyconfigs/pull/20389 diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-rebuild-z3-with-python-bindings-sapphirerapids.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-rebuild-z3-with-python-bindings-sapphirerapids.yml new file mode 100644 index 0000000000..a00c244a41 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.2-rebuild-z3-with-python-bindings-sapphirerapids.yml @@ -0,0 +1,8 @@ +# 2025.02.28 +# Z3/4.12.2-GCCcore-12.3.0 rebuild to account for easyconfig changed upstream +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/20050 +easyconfigs: + - Z3-4.12.2-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20050 + from-commit: 985b515a0bff531bdcafc5b83eb4160537c5db2c diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-R-bundle-Bioconductor-additional-extensions.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-R-bundle-Bioconductor-additional-extensions.yml new file mode 100644 index 0000000000..2ed6f0a363 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-R-bundle-Bioconductor-additional-extensions.yml @@ -0,0 +1,9 @@ +# 2025.02.28 +# R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb rebuild to account for easyconfig changed upstream +# (additional extensions have been added) +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/21948 +easyconfigs: + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21948 + from-commit: f9cfe6ac7d9019970c2be3e8b09db4d846cf005a diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-scikit-build-core-add-pyproject-metadata.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-scikit-build-core-add-pyproject-metadata.yml new file mode 100644 index 0000000000..7b42156834 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250228-eb-4.9.4-scikit-build-core-add-pyproject-metadata.yml @@ -0,0 +1,8 @@ +# 2025.02.28 +# scikit-build-core-0.9.3-GCCcore-13.2.0.eb to account for easyconfig changed upstream +# see https://github.com/easybuilders/easybuild-easyconfigs/pull/21671 +easyconfigs: + - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21671 + from-commit: c38f0637504bcd66e6f7f80277552934e1b03127 diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20250410-eb-5.0.0-siesta-rebuild-with-controlled-optimisations.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20250410-eb-5.0.0-siesta-rebuild-with-controlled-optimisations.yml new file mode 100644 index 0000000000..e608c69f40 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20250410-eb-5.0.0-siesta-rebuild-with-controlled-optimisations.yml @@ -0,0 +1,8 @@ +# 2025.04.10 +# Siesta-5.2.2-foss-2023a.eb easyconfig changed upstream to only use +# eb-sourced compilation options +easyconfigs: + - Siesta-5.2.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22731 + from-commit: b5e36fe564794234cf2d79621ad2d61c3980e5a7 diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml deleted file mode 100644 index f261e467b1..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2022b.yml +++ /dev/null @@ -1,14 +0,0 @@ -easyconfigs: - - GCC-12.2.0.eb - - Python-3.10.8-GCCcore-12.2.0-bare.eb - - make-4.3-GCCcore-12.2.0.eb - - cairo-1.17.4-GCCcore-12.2.0.eb - - UCC-1.1.0-GCCcore-12.2.0.eb - - PMIx-4.2.2-GCCcore-12.2.0.eb - - libfabric-1.16.1-GCCcore-12.2.0.eb - - foss-2022b.eb - - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: - options: - from-pr: 19339 - - Qt5-5.15.7-GCCcore-12.2.0.eb - - QuantumESPRESSO-7.2-foss-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml deleted file mode 100644 index 19f80d71fe..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023a.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - GCCcore-12.3.0.eb: - options: - from-pr: 20218 - - OpenMPI-4.1.5-GCC-12.3.0: - options: - from-pr: 19940 - - METIS-5.1.0-GCCcore-12.3.0.eb - - SCOTCH-7.0.3-gompi-2023a.eb - - CGAL-5.6-GCCcore-12.3.0.eb - - ParaView-5.11.2-foss-2023a.eb - - gnuplot-5.4.8-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml deleted file mode 100644 index 57eda0aec4..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023a.yml +++ /dev/null @@ -1,4 +0,0 @@ -easyconfigs: - - Python-3.11.3-GCCcore-12.3.0.eb - - hatchling-1.18.0-GCCcore-12.3.0.eb - - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml deleted file mode 100644 index 8ec0067465..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2022b.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml deleted file mode 100644 index 7257ec4dae..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023a.yml +++ /dev/null @@ -1,12 +0,0 @@ -easyconfigs: - - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb - - at-spi2-core-2.49.91-GCCcore-12.3.0.eb - - OpenFOAM-10-foss-2023a.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 - from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f - - OpenFOAM-11-foss-2023a.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 - from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f - diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml deleted file mode 100644 index 4d9f5411ed..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-2023b.yml +++ /dev/null @@ -1,3 +0,0 @@ -easyconfigs: - - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb - - at-spi2-core-2.50.0-GCCcore-13.2.0.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml deleted file mode 100644 index ac5139290c..0000000000 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-2022b.yml +++ /dev/null @@ -1,5 +0,0 @@ -easyconfigs: - - OpenBLAS-0.3.21-GCC-12.2.0.eb: - options: - # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 - include-easyblocks-from-commit: d06d9617d9bfb63d338b6879eab9da81c8a312d8 diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-001-system.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-001-system.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 100644 index 0000000000..c3c70a7585 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1,17 @@ +easyconfigs: + - OpenBLAS-0.3.21-GCC-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19159 + # required for Sapphire Rapids support + from-pr: 19159 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3492 + include-easyblocks-from-pr: 3492 + - OpenMPI-4.1.4-GCC-12.2.0.eb: + options: + from-pr: 19940 + - foss-2022b.eb + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: + options: + from-pr: 19339 + - Qt5-5.15.7-GCCcore-12.2.0.eb + - QuantumESPRESSO-7.2-foss-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2023a.yml similarity index 84% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2023a.yml index bd34313a10..bb53c2b19e 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.8.2-2023a.yml @@ -1,21 +1,20 @@ easyconfigs: - - XZ-5.4.2-GCCcore-12.3.0.eb - - libffi-3.4.4-GCCcore-12.3.0.eb - - SQLite-3.42.0-GCCcore-12.3.0.eb - - bzip2-1.0.8-GCCcore-12.3.0.eb - - UnZip-6.0-GCCcore-12.3.0.eb - - cairo-1.17.8-GCCcore-12.3.0.eb - - Rust-1.70.0-GCCcore-12.3.0.eb: + - GCCcore-12.3.0.eb: options: - include-easyblocks-from-pr: 3038 - - poetry-1.5.1-GCCcore-12.3.0.eb - - git-2.41.0-GCCcore-12.3.0-nodocs.eb - - flit-3.9.0-GCCcore-12.3.0.eb + from-pr: 20218 - OpenBLAS-0.3.23-GCC-12.3.0.eb: options: # required for Intel Sapphire Rapids support # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19159 from-pr: 19159 + - Rust-1.70.0-GCCcore-12.3.0.eb: + # fix build of Rust 1.70.0 by disabling download of pre-built LLVM; + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3038 + options: + include-easyblocks-from-pr: 3038 + - OpenMPI-4.1.5-GCC-12.3.0: + options: + from-pr: 19940 - foss-2023a.eb - pybind11-2.11.1-GCCcore-12.3.0.eb: # avoid indirect dependency on old CMake version built with GCCcore/10.2.0 via Catch2 build dependency; diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-001-system.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-001-system.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2022b.yml similarity index 92% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2022b.yml index a8dc0693a3..2ec407636a 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2022b.yml @@ -1,5 +1,4 @@ easyconfigs: - - OpenMPI-4.1.4-GCC-12.2.0.eb - Highway-1.0.3-GCCcore-12.2.0.eb: options: from-pr: 20298 diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023a.yml new file mode 100644 index 0000000000..201015c371 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023a.yml @@ -0,0 +1,86 @@ +easyconfigs: + - METIS-5.1.0-GCCcore-12.3.0.eb + - SCOTCH-7.0.3-gompi-2023a.eb + - CGAL-5.6-GCCcore-12.3.0.eb + - ParaView-5.11.2-foss-2023a.eb + - gnuplot-5.4.8-GCCcore-12.3.0.eb + - ESPResSo-4.2.1-foss-2023a.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19592 + options: + from-pr: 19592 + - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb: + options: + from-pr: 19679 + - Pillow-10.0.0-GCCcore-12.3.0.eb + - sympy-1.12-gfbf-2023a.eb + - networkx-3.1-gfbf-2023a.eb + - expecttest-0.1.5-GCCcore-12.3.0.eb + - PyYAML-6.0-GCCcore-12.3.0.eb + - pytest-flakefinder-1.1.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19480 + from-pr: 19480 + - pytest-rerunfailures-12.0-GCCcore-12.3.0.eb + - pytest-shard-0.1.2-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19480 + from-pr: 19480 + - scikit-learn-1.3.1-gfbf-2023a.eb + - snakemake-8.4.2-foss-2023a.eb: + options: + from-pr: 19646 + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19471 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3036 + options: + from-pr: 19471 + include-easyblocks-from-pr: 3036 + - PyTorch-2.1.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19573 + from-pr: 19573 + - matplotlib-3.7.2-gfbf-2023a.eb + - PyQt5-5.15.10-GCCcore-12.3.0.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554 + options: + from-pr: 19554 + - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 + from-pr: 19996 + - dask-2023.9.2-foss-2023a.eb + - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb + - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: + options: + from-pr: 20086 + - Z3-4.12.2-GCCcore-12.3.0.eb: + options: + # The Z3 dependency of PyTorch had it's versionsuffix removed + # and we need to workaround the problem this creates, + # see https://github.com/EESSI/software-layer/pull/501 for details + from-pr: 20050 + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb: + options: + from-pr: 20007 + - OpenJPEG-2.5.0-GCCcore-12.3.0.eb + - Highway-1.0.4-GCCcore-12.3.0.eb + - ELPA-2023.05.001-foss-2023a.eb + - libxc-6.2.2-GCC-12.3.0.eb + - SuperLU_DIST-8.1.2-foss-2023a.eb: + options: + from-pr: 20162 + - PETSc-3.20.3-foss-2023a.eb: + options: + include-easyblocks-from-pr: 3086 + from-pr: 19686 + - MODFLOW-6.4.4-foss-2023a.eb: + options: + from-pr: 20142 + # add all dependencies of R-bundle-CRAN-2023.12-foss-2023a.eb; due to a rebuild we will install the bundle itself with EB 4.9.4 + - NLopt-2.7.1-GCCcore-12.3.0.eb + - nettle-3.9.1-GCCcore-12.3.0.eb + - Xvfb-21.1.8-GCCcore-12.3.0.eb + - libsndfile-1.2.2-GCCcore-12.3.0.eb + - PostgreSQL-16.1-GCCcore-12.3.0.eb + - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb + - GDAL-3.7.1-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023b.yml similarity index 85% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023b.yml index 23fc934645..a0e744c5a3 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.0-2023b.yml @@ -2,18 +2,10 @@ easyconfigs: - GCCcore-13.2.0.eb: options: from-pr: 19974 + - GCC-13.2.0.eb - OpenMPI-4.1.6-GCC-13.2.0: options: from-pr: 19940 - - XZ-5.4.4-GCCcore-13.2.0.eb - - SQLite-3.43.1-GCCcore-13.2.0.eb - - UnZip-6.0-GCCcore-13.2.0.eb - - libffi-3.4.4-GCCcore-13.2.0.eb - - cairo-1.18.0-GCCcore-13.2.0.eb - - poetry-1.6.1-GCCcore-13.2.0.eb - - git-2.42.0-GCCcore-13.2.0.eb - - flit-3.9.0-GCCcore-13.2.0.eb - - expat-2.5.0-GCCcore-13.2.0.eb - foss-2023b.eb - SciPy-bundle-2023.11-gfbf-2023b.eb - netCDF-4.9.2-gompi-2023b.eb: diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2022b.yml new file mode 100644 index 0000000000..1805c581c3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2022b.yml @@ -0,0 +1,8 @@ +easyconfigs: + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20379 + - ParaView-5.11.1-foss-2022b.eb + - ASE-3.22.1-gfbf-2022b.eb + - SEPP-4.5.1-foss-2022b.eb + - Valgrind-3.21.0-gompi-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023a.yml new file mode 100644 index 0000000000..1df15046d3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023a.yml @@ -0,0 +1,19 @@ +easyconfigs: + - ncdu-1.18-GCC-12.3.0.eb + - SAMtools-1.18-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 20379 + - ipympl-0.9.3-gfbf-2023a.eb: + options: + from-pr: 18852 + - ESPResSo-4.2.2-foss-2023a.eb: + options: + from-pr: 20595 + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - WhatsHap-2.2-foss-2023a.eb + - BLAST+-2.14.1-gompi-2023a.eb: + options: + from-pr: 20784 + - Valgrind-3.21.0-gompi-2023a.eb + - OrthoFinder-2.5.5-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023b.yml similarity index 76% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023b.yml index 987d8c2625..e6f13edc05 100644 --- a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.1-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.1-2023b.yml @@ -1,7 +1,5 @@ easyconfigs: - - Python-3.11.5-GCCcore-13.2.0.eb - - hatchling-1.18.0-GCCcore-13.2.0.eb - - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + # include the changes from rebuilds/20240515-eb-4.9.1-GROMACS-correct-gmxapi-version.yml - scikit-build-core-0.9.3-GCCcore-13.2.0.eb: options: # from-commit: 61d07bff09afe63cfe1ae35dc58a0c8be01eed62 diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-001-system.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.2-001-system.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2022b.yml new file mode 100644 index 0000000000..969b0d469b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2022b.yml @@ -0,0 +1,52 @@ +easyconfigs: + - BLAST+-2.14.0-gompi-2022b.eb + - BioPerl-1.7.8-GCCcore-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - h5py-3.8.0-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb + - MUMPS-5.6.1-foss-2022b-metis.eb + - GL2PS-1.4.2-GCCcore-12.2.0.eb + - GST-plugins-base-1.22.1-GCC-12.2.0.eb + - wxWidgets-3.2.2.1-GCC-12.2.0.eb + - Archive-Zip-1.68-GCCcore-12.2.0.eb + - jemalloc-5.3.0-GCCcore-12.2.0.eb + - Judy-1.0.5-GCCcore-12.2.0.eb + - libaio-0.3.113-GCCcore-12.2.0.eb + - Z3-4.12.2-GCCcore-12.2.0.eb + - tbb-2021.10.0-GCCcore-12.2.0.eb + - dask-2023.7.1-foss-2022b.eb + - netcdf4-python-1.6.3-foss-2022b.eb + - Ruby-3.2.2-GCCcore-12.2.0.eb + - ROOT-6.26.10-foss-2022b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023a.yml new file mode 100644 index 0000000000..cd8684a935 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023a.yml @@ -0,0 +1,86 @@ +easyconfigs: + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - BCFtools-1.18-GCC-12.3.0.eb + - BWA-0.7.18-GCCcore-12.3.0.eb + - CapnProto-1.0.1-GCCcore-12.3.0.eb + - DendroPy-4.6.1-GCCcore-12.3.0.eb + - DIAMOND-2.1.8-GCC-12.3.0.eb + - FastME-2.1.6.3-GCC-12.3.0.eb + - fastp-0.23.4-GCC-12.3.0.eb + - HMMER-3.4-gompi-2023a.eb + - IQ-TREE-2.3.5-gompi-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20955 + from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784 + - KronaTools-2.8.1-GCCcore-12.3.0.eb + - LSD2-2.4.1-GCCcore-12.3.0.eb + - MAFFT-7.520-GCC-12.3.0-with-extensions.eb + - ncbi-vdb-3.0.10-gompi-2023a.eb + - MetalWalls-21.06.1-foss-2023a.eb + - QuantumESPRESSO-7.3.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 + from-commit: dbdaacc0739fdee91baa9123864ea4428cf21273 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 + include-easyblocks-from-commit: 32e45bd1f2d916732ca5852d55d17fa4d99e388b + - CP2K-2023.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 + - amdahl-0.3.1-gompi-2023a.eb + - LLVM-14.0.6-GCCcore-12.3.0-llvmlite.eb + - numba-0.58.1-foss-2023a.eb + - librosa-0.10.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21434 + from-commit: 4b6170603150c9c93602b0deb726869d19a390f0 + - xarray-2023.9.0-gfbf-2023a.eb + - SciTools-Iris-3.9.0-foss-2023a.eb + - OpenFOAM-v2312-foss-2023a.eb: + options: + # https://github.com/easybuilders/easybuild-easyblocks/pull/3388 + include-easyblocks-from-commit: c8256a36e7062bc09f5ce30552a9de9827054c9e + # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841 + from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649 + - BioPerl-1.7.8-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - grpcio-1.57.0-GCCcore-12.3.0.eb + - orjson-3.9.15-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880 + from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90 + - wradlib-2.0.3-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21094 + from-commit: 3a2e0b8e6ee45277d01fb7e2eb93027a28c9461a + - MBX-1.1.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21155 + from-commit: 6929a67401f2a2ec58f91fb306332a77497d73ff + - Transrate-1.0.3-GCC-12.3.0.eb: + options: + # https://github.com/easybuilders/easybuild-easyblocks/pull/3381 + include-easyblocks-from-commit: bb86f05d4917b29e022023f152efdf0ca5c14ded + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20964 + from-commit: 7d539a9e599d8bc5ac2bda6ee9587ef62351ee03 + - Critic2-1.2-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20833 + from-commit: 78426c2383fc7e4b9b9e77d7a77f336e1bee3843 + - LRBinner-0.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21310 + from-commit: 799d9101df2cf81aabe252f00cc82a7246363f53 + - Redland-1.0.17-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21227 + from-commit: 4c5e3455dec31e68e8383c7fd86d1f80c434676d diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023b.yml new file mode 100644 index 0000000000..f4a9d5e493 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.2-2023b.yml @@ -0,0 +1,27 @@ +easyconfigs: + - IPython-8.17.2-GCCcore-13.2.0.eb + - dlb-3.4-gompi-2023b.eb + - pystencils-1.3.4-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 + from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 + - Extrae-4.2.0-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21017 + from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393 + include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c + - pyMBE-0.8.0-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 + from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 + - STAR-2.7.11b-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200 + from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00 + - HPL-2.3-foss-2023b.eb + - R-bundle-CRAN-2024.06-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21366 + # we use a commit from the Brunsli PR here to get rid of the Highway dependency + from-commit: 1736a123b1685836452587a5c51793257570bb2d diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..ec601e2e4b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,8 @@ +easyconfigs: + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb + - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..2cf72ca098 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,11 @@ +easyconfigs: + - GROMACS-2024.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21430 + from-commit: 8b509882d03402e2998ff9b22c154a6957e36d6b + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3569 + include-easyblocks-from-commit: 362b4679193612e04abe336fa041e2a34d183991 diff --git a/easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-001-system.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/sapphire_rapids/eessi-2023.06-eb-4.9.4-001-system.yml rename to easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..7e920d3dda --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,53 @@ +easyconfigs: + - R-bundle-CRAN-2023.12-foss-2023a.eb + - archspec-0.2.1-GCCcore-12.3.0.eb + - ROOT-6.30.06-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 + - waLBerla-6.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21600 + from-commit: 9b12318bcff1749781d9eb71c23e21bc3a79ed01 + - mpl-ascii-0.10.0-gfbf-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21679 + from-commit: 7106f63160b1418d605882dd02ba151d099300bd + - jedi-0.19.0-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21650 + from-commit: 109998f6adcda7efb4174b1e5f73b41ee82d1f13 + - Solids4foam-2.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21606 + from-commit: 63562c58acf1be64407192b6862c3bd80253d2e0 + - Cassiopeia-2.0.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21657 + from-commit: 7f1f0e60487e7e1fcb5c4e6bc4fbc4f89994e3fd + - LightGBM-4.5.0-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21699 + from-commit: e3407bd127d248c08960f6b09c973da0fdecc2c3 + - OpenFOAM-v2406-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3519 + include-easyblocks-from-commit: e4a3ff1932350d575dffc7597435609fad6dd691 + - Paraver-4.11.4-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20230 + from-commit: 91c8df6b4c0810061e9f325427c9c79e961bc4b0 + - Tombo-1.5.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21925 + from-commit: 522ca010ab11949ab9594037f72b975cf1cd0d33 + - elfx86exts-0.6.2-GCC-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22145 + from-commit: 31478e5c9869de3add74d0a02dd5df01ea65b21e + - archspec-0.2.5-GCCcore-12.3.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/22235 + from-commit: 01dd97ea62fe4d7d0df040ede3af03eb2f1b8641 diff --git a/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023b.yml b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023b.yml new file mode 100644 index 0000000000..0ff41ec4fa --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/sapphirerapids/eessi-2023.06-eb-4.9.4-2023b.yml @@ -0,0 +1,39 @@ +easyconfigs: + - SIONlib-1.7.7-GCCcore-13.2.0-tools.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21752 + from-commit: 6b8b53493a1188a5baa56a133574daac239730e7 + - Score-P-8.4-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3496 + include-easyblocks-from-commit: 60633b0acfd41a0732992d9e16800dae71a056eb + - Cython-3.0.10-GCCcore-13.2.0.eb + - Mustache-1.3.3-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21783 + from-commit: 5fa3db9eb36f91cba3fbf351549f8ba2849abc33 + - GDRCopy-2.4-GCCcore-13.2.0.eb + - GROMACS-2024.4-foss-2023b.eb: + options: + # https://github.com/easybuilders/easybuild-easyconfigs/pull/21851 + from-commit: f0fa64b440deaf5fb0a6d26ff1bb3e9f36626c8a + - SlurmViewer-1.0.1-GCCcore-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21899 + from-commit: 0bdeb23c9ea5a3caefd353ecd936919424c1bba4 + - wxWidgets-3.2.6-GCC-13.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21915 + from-commit: 58f16c0caf8c5494c68e9eda8cbf19e9145d3cfa + - DP3-6.2-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - WSClean-3.5-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 + - EveryBeam-0.6.1-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21765 + from-commit: c7e4bfe1a57cf9781ce346ba8ae9081644408c23 diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml new file mode 100644 index 0000000000..ddc783d11a --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.8.2-2022b.yml @@ -0,0 +1,11 @@ +easyconfigs: + # Adding OpenMPI explicitely, as defined in 20240301-eb-4.9.0-OpenMPI-4.1.x-fix-smcuda.yml + - OpenMPI-4.1.4-GCC-12.2.0.eb: + options: + from-pr: 19940 + - foss-2022b.eb + - HarfBuzz-5.3.1-GCCcore-12.2.0.eb: + options: + from-pr: 19339 + - Qt5-5.15.7-GCCcore-12.2.0.eb + - QuantumESPRESSO-7.2-foss-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml new file mode 100644 index 0000000000..763fcb20a8 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.0-2022b.yml @@ -0,0 +1,28 @@ +easyconfigs: + # Adding Python explicitely, as defined in 20240729-eb-4.9.2-Python-ctypes.yml + - Python-3.10.8-GCCcore-12.2.0-bare: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + # Can't use include-easyblocks-from-commit here, as was done for rebuilds for the other archs + # because the eb version from this easystack isn't new enough to know it + # include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + include-easyblocks-from-pr: 3352 + - Python-3.10.8-GCCcore-12.2.0: + options: + # See https://github.com/easybuilders/easybuild-easyblocks/pull/3352 + # Can't use include-easyblocks-from-commit here, as was done for rebuilds for the other archs + # because the eb version from this easystack isn't new enough to know it + # include-easyblocks-from-commit: 1ee17c0f7726c69e97442f53c65c5f041d65c94f + include-easyblocks-from-pr: 3352 + - SciPy-bundle-2023.02-gfbf-2022b.eb + - GDAL-3.6.2-foss-2022b.eb + - waLBerla-6.1-foss-2022b.eb: + options: + from-pr: 19324 + - WRF-4.4.1-foss-2022b-dmpar.eb + - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb: + options: + from-pr: 20086 + - R-4.2.2-foss-2022b.eb: + options: + from-pr: 20238 diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml new file mode 100644 index 0000000000..1805c581c3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-2022b.yml @@ -0,0 +1,8 @@ +easyconfigs: + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20379 + - ParaView-5.11.1-foss-2022b.eb + - ASE-3.22.1-gfbf-2022b.eb + - SEPP-4.5.1-foss-2022b.eb + - Valgrind-3.21.0-gompi-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml new file mode 100644 index 0000000000..969b0d469b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2022b.yml @@ -0,0 +1,52 @@ +easyconfigs: + - BLAST+-2.14.0-gompi-2022b.eb + - BioPerl-1.7.8-GCCcore-12.2.0.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136 + from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc + - gnuplot-5.4.6-GCCcore-12.2.0.eb + - h5py-3.8.0-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb + - MUMPS-5.6.1-foss-2022b-metis.eb + - GL2PS-1.4.2-GCCcore-12.2.0.eb + - GST-plugins-base-1.22.1-GCC-12.2.0.eb + - wxWidgets-3.2.2.1-GCC-12.2.0.eb + - Archive-Zip-1.68-GCCcore-12.2.0.eb + - jemalloc-5.3.0-GCCcore-12.2.0.eb + - Judy-1.0.5-GCCcore-12.2.0.eb + - libaio-0.3.113-GCCcore-12.2.0.eb + - Z3-4.12.2-GCCcore-12.2.0.eb + - tbb-2021.10.0-GCCcore-12.2.0.eb + - dask-2023.7.1-foss-2022b.eb + - netcdf4-python-1.6.3-foss-2022b.eb + - Ruby-3.2.2-GCCcore-12.2.0.eb + - ROOT-6.26.10-foss-2022b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 diff --git a/eb_hooks.py b/eb_hooks.py index 08eb5ea62e..4cf4b7a1fb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -25,7 +25,9 @@ CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1' CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' CPU_TARGET_A64FX = 'aarch64/a64fx' +CPU_TARGET_NVIDIA_GRACE = 'aarch64/nvidia/grace' +CPU_TARGET_SAPPHIRE_RAPIDS = 'x86_64/intel/sapphirerapids' CPU_TARGET_ZEN4 = 'x86_64/amd/zen4' EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -500,7 +502,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg # note: keep condition in sync with the one used in # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': - if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]: + if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1, CPU_TARGET_NVIDIA_GRACE]: self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF') if cpu_target == CPU_TARGET_NEOVERSE_N1: self.orig_optarch = build_option('optarch') @@ -628,7 +630,7 @@ def pre_configure_hook_gromacs(self, *args, **kwargs): """ if self.name == 'GROMACS': cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: + if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1 or LooseVersion(self.version) <= LooseVersion('2024.4') and CPU_TARGET_NVIDIA_GRACE: self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') print_msg( "Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", @@ -747,10 +749,17 @@ def pre_test_hook_exclude_failing_test_Highway(self, *args, **kwargs): """ Pre-test hook for Highway: exclude failing TestAllShiftRightLanes/SVE_256 test on neoverse_v1 cfr. https://github.com/EESSI/software-layer/issues/469 + and exclude failing tests + HwyReductionTestGroup/HwyReductionTest.TestAllSumOfLanes/SVE2_128 + HwyReductionTestGroup/HwyReductionTest.TestAllSumOfLanes/SVE2 + HwyReductionTestGroup/HwyReductionTest.TestAllSumOfLanes/SVE + on nvidia/grace """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'Highway' and self.version in ['1.0.3'] and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['runtest'] += ' ARGS="-E TestAllShiftRightLanes/SVE_256"' + if self.name == 'Highway' and self.version in ['1.0.3'] and cpu_target == CPU_TARGET_NVIDIA_GRACE: + self.cfg['runtest'] += ' ARGS="-E TestAllSumOfLanes"' def pre_test_hook_ignore_failing_tests_ESPResSo(self, *args, **kwargs): @@ -789,16 +798,71 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris FAILED scipy/spatial/tests/test_distance.py::TestPdist::test_pdist_correlation_iris_float32 = 4 failed, 54407 passed, 3016 skipped, 223 xfailed, 13 xpassed, 10917 warnings in 6068.43s (1:41:08) = + In version 2023.07 + 2023.11 on grace, 2 failing tests in scipy (versions 1.11.1, 1.11.4): + FAILED scipy/optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 + FAILED scipy/optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 + = 2 failed, 54876 passed, 3021 skipped, 223 xfailed, 13 xpassed in 581.85s (0:09:41) = + In version 2023.02 on grace, 46 failing tests in scipy (versions 1.10.1): + FAILED ../../linalg/tests/test_basic.py::TestOverwrite::test_pinv - RuntimeWa... + FAILED ../../linalg/tests/test_basic.py::TestOverwrite::test_pinvh - RuntimeW... + FAILED ../../linalg/tests/test_matfuncs.py::TestExpM::test_2x2_input - Runtim... + FAILED ../../optimize/tests/test_linprog.py::TestLinprogIPSparse::test_bug_6139 + FAILED ../../optimize/tests/test_linprog.py::TestLinprogIPSparsePresolve::test_bug_6139 + FAILED ../../optimize/tests/test_zeros.py::test_gh_9608_preserve_array_shape + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[True-True-True-coo_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[True-True-False-array-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[True-True-False-csr_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[True-True-False-coo_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[False-True-True-array-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[False-True-True-csr_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[False-True-True-coo_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[True-True-True-array-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-True-False-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[False-True-False-array-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[True-True-True-csr_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-True-False-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-True-True-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-True-True-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-True-False-True-float32-coo_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[False-True-False-csr_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-True-True-True-float32-coo_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-True-False-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-False-False-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-False-False-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-False-False-True-float32-coo_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_asymmetric_laplacian[False-True-False-coo_matrix-float32] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-True-False-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-True-False-True-float32-coo_matrix] + FAILED ../../sparse/linalg/_eigen/lobpcg/tests/test_lobpcg.py::test_tolerance_float32 + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-True-True-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-False-False-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-False-False-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-False-False-True-float32-coo_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-True-True-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-True-True-True-float32-coo_matrix] + FAILED ../../sparse/linalg/_eigen/lobpcg/tests/test_lobpcg.py::test_random_initial_float32 + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-False-True-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-False-True-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[function-False-True-True-float32-coo_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-False-True-True-float32-asarray] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-False-True-True-float32-csr_matrix] + FAILED ../../sparse/csgraph/tests/test_graph_laplacian.py::test_format[lo-False-True-True-float32-coo_matrix] + FAILED ../../sparse/linalg/_isolve/tests/test_iterative.py::test_precond_dummy + FAILED ../../sparse/linalg/_eigen/arpack/tests/test_arpack.py::test_symmetric_modes + = 46 failed, 49971 passed, 2471 skipped, 231 xfailed, 11 xpassed in 65.91s (0:01:05) = (in previous versions we were not as strict yet on the numpy/SciPy tests) """ cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') scipy_bundle_versions_nv1 = ('2021.10', '2023.02', '2023.07', '2023.11') scipy_bundle_versions_a64fx = ('2023.07', '2023.11') + scipy_bundle_versions_nvidia_grace = ('2023.02', '2023.07', '2023.11') if self.name == 'SciPy-bundle': if cpu_target == CPU_TARGET_NEOVERSE_V1 and self.version in scipy_bundle_versions_nv1: self.cfg['testopts'] = "|| echo ignoring failing tests" elif cpu_target == CPU_TARGET_A64FX and self.version in scipy_bundle_versions_a64fx: self.cfg['testopts'] = "|| echo ignoring failing tests" + elif cpu_target == CPU_TARGET_NVIDIA_GRACE and self.version in scipy_bundle_versions_nvidia_grace: + self.cfg['testopts'] = "|| echo ignoring failing tests" def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): @@ -817,11 +881,16 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ - Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 - See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 + Pre-test hook for PyTorch: increase max failing tests for ARM and Intel Sapphire Rapids for PyTorch 2.1.2 + See https://github.com/EESSI/software-layer/pull/444#issuecomment-1890416171 and + https://github.com/EESSI/software-layer/pull/875#issuecomment-2606854400 """ - if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64: - self.cfg['max_failed_tests'] = 10 + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'PyTorch' and self.version == '2.1.2': + if get_cpu_architecture() == AARCH64: + self.cfg['max_failed_tests'] = 10 + if cpu_target == CPU_TARGET_SAPPHIRE_RAPIDS: + self.cfg['max_failed_tests'] = 4 def pre_single_extension_hook(ext, *args, **kwargs): diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index c5cdc68941..28cc724ada 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -56,3 +56,14 @@ - SciPy-bundle-2023.11-gfbf-2023b: - issue: https://github.com/EESSI/software-layer/issues/318 - info: "2 failing tests (vs 54876 passed) in scipy test suite" +- aarch64/nvidia/grace: + - Highway-1.0.4-GCCcore-12.3.0.eb: + - issue: https://github.com/EESSI/software-layer/issues/469 + - info: "failing to build the tests" + - SciPy-bundle-2023.07-gfbf-2023a: + - issue: https://github.com/EESSI/software-layer/issues/318 + - info: "2 failing tests (vs 54409 passed) in scipy test suite" +- x86_64/intel/sapphire_rapids: + - PyTorch-2.1.2-foss-2023a: + - issue: https://github.com/EESSI/software-layer/issues/461 + - info: "4 failing tests (out of 209567) on x86_64/intel/sapphire_rapids" diff --git a/eessi_container.sh b/eessi_container.sh index 9de2cb37e9..3edddf6c46 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -89,6 +89,8 @@ display_help() { echo " -n | --nvidia MODE - configure the container to work with NVIDIA GPUs," echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" + echo " -p | --pass-through ARG - argument to pass through to the launch of the" + echo " container; can be given multiple times [default: not set]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use; can be given multiple times;" echo " CFG may include a suffix ',access={ro,rw}' to" @@ -126,6 +128,7 @@ VERBOSE=0 STORAGE= LIST_REPOS=0 MODE="shell" +PASS_THROUGH=() SETUP_NVIDIA=0 REPOSITORIES=() RESUME= @@ -182,6 +185,10 @@ while [[ $# -gt 0 ]]; do NVIDIA_MODE="$2" shift 2 ;; + -p|--pass-through) + PASS_THROUGH+=("$2") + shift 2 + ;; -r|--repository) REPOSITORIES+=("$2") shift 2 @@ -401,6 +408,19 @@ else echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')." fi +# if ${RESUME} is a file, unpack it into ${EESSI_HOST_STORAGE} +if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then + if [[ "${RESUME}" == *.tgz ]]; then + tar xf ${RESUME} -C ${EESSI_HOST_STORAGE} + # Add support for resuming from zstd-compressed tarballs + elif [[ "${RESUME}" == *.zst && -x "$(command -v zstd)" ]]; then + zstd -dc ${RESUME} | tar -xf - -C ${EESSI_HOST_STORAGE} + elif [[ "${RESUME}" == *.zst && ! -x "$(command -v zstd)" ]]; then + fatal_error "Trying to resume from tarball ${RESUME} which was compressed using zstd, but zstd command not found" + fi + echo "Resuming from previous run using temporary storage ${RESUME} unpacked into ${EESSI_HOST_STORAGE}" +fi + # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then tar xf ${RESUME} -C ${EESSI_HOST_STORAGE} @@ -529,6 +549,15 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" + +# if TMPDIR is not empty and if TMP_IN_CONTAINER is not a prefix of TMPDIR, we need to add a bind mount for TMPDIR +if [[ ! -z ${TMPDIR} && ${TMP_IN_CONTAINER} != ${TMPDIR}* ]]; then + msg="TMPDIR is not empty (${TMPDIR}) and TMP_IN_CONTAINER (${TMP_IN_CONTAINER}) is not a prefix of TMPDIR:" + msg="${msg} adding bind mount for TMPDIR" + echo "${msg}" + BIND_PATHS="${BIND_PATHS},${TMPDIR}" +fi + if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then BIND_PATHS="${BIND_PATHS},${EXTRA_BIND_PATHS}" fi @@ -706,8 +735,12 @@ fi declare -a EESSI_FUSE_MOUNTS=() -# always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) -EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") +# mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) unless env var +# EESSI_DO_NOT_MOUNT_CVMFS_CONFIG_CERN_CH is defined +if [ -z ${EESSI_DO_NOT_MOUNT_CVMFS_CONFIG_CERN_CH+x} ]; then + EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") +fi + # iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) for cvmfs_repo in "${REPOSITORIES[@]}" @@ -837,6 +870,11 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +# add pass through arguments +for arg in "${PASS_THROUGH[@]}"; do + ADDITIONAL_CONTAINER_OPTIONS+=(${arg}) +done + echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${ADDITIONAL_CONTAINER_OPTIONS[@]} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" singularity ${RUN_QUIET} ${MODE} "${ADDITIONAL_CONTAINER_OPTIONS[@]}" "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" @@ -848,17 +886,30 @@ if [[ ! -z ${SAVE} ]]; then # ARCH which might have been used internally, eg, when software packages # were built ... we rather keep the script here "stupid" and leave the handling # of these aspects to where the script is used + # Compression with zlib may be quite slow. On some systems, the pipeline takes ~20 mins for a 2 min build because of this. + # Check if zstd is present for faster compression and decompression if [[ -d ${SAVE} ]]; then # assume SAVE is name of a directory to which tarball shall be written to # name format: tmp_storage-{TIMESTAMP}.tgz ts=$(date +%s) - TGZ=${SAVE}/tmp_storage-${ts}.tgz + if [[ -x "$(command -v zstd)" ]]; then + TARBALL=${SAVE}/tmp_storage-${ts}.zst + tar -cf - -C ${EESSI_TMPDIR} . | zstd -T0 > ${TARBALL} + else + TARBALL=${SAVE}/tmp_storage-${ts}.tgz + tar czf ${TARBALL} -C ${EESSI_TMPDIR} . + fi else # assume SAVE is the full path to a tarball's name - TGZ=${SAVE} + TARBALL=${SAVE} + # if zstd is present and a .zst extension is asked for, use it + if [[ "${SAVE}" == *.zst && -x "$(command -v zstd)" ]]; then + tar -cf - -C ${EESSI_TMPDIR} . | zstd -T0 > ${TARBALL} + else + tar czf ${TARBALL} -C ${EESSI_TMPDIR} + fi fi - tar czf ${TGZ} -C ${EESSI_TMPDIR} . - echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" + echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TARBALL}' (to resume session add '--resume ${TARBALL}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) diff --git a/init/arch_specs/eessi_arch_x86.spec b/init/arch_specs/eessi_arch_x86.spec index bfbc5b4be1..46289bd4a9 100755 --- a/init/arch_specs/eessi_arch_x86.spec +++ b/init/arch_specs/eessi_arch_x86.spec @@ -1,7 +1,8 @@ # x86_64 CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell -"x86_64/intel/skylake_avx512" "GenuineIntel" "avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl" # Intel Skylake, Cascade Lake -"x86_64/amd/zen2" "AuthenticAMD" "avx2 fma" # AMD Rome -"x86_64/amd/zen3" "AuthenticAMD" "avx2 fma vaes" # AMD Milan, Milan-X -"x86_64/amd/zen4" "AuthenticAMD" "avx2 fma vaes avx512f avx512ifma" # AMD Genoa, Genoa-X +"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell +"x86_64/intel/skylake_avx512" "GenuineIntel" "avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl" # Intel Skylake, Cascade Lake +"x86_64/intel/sapphirerapids" "GenuineIntel" "avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl avx512_bf16 amx_tile" # Intel Sapphire/Emerald Rapids +"x86_64/amd/zen2" "AuthenticAMD" "avx2 fma" # AMD Rome +"x86_64/amd/zen3" "AuthenticAMD" "avx2 fma vaes" # AMD Milan, Milan-X +"x86_64/amd/zen4" "AuthenticAMD" "avx2 fma vaes avx512f avx512ifma" # AMD Genoa, Genoa-X diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 60d69cc198..0f13b1493a 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -87,16 +87,6 @@ if [ -d $EESSI_PREFIX ]; then fi if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then - # use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, - # since optimized software installations for Zen4 are a work-in-progress, - # see https://gitlab.com/eessi/support/-/issues/37 - if [[ "${EESSI_SOFTWARE_SUBDIR}" == "x86_64/amd/zen4" ]]; then - if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen3" - echo -e "\e[33mSticking to ${EESSI_SOFTWARE_SUBDIR} for now, since optimized installations for AMD Genoa (Zen4) are a work in progress, see https://gitlab.com/eessi/support/-/issues/37 for more information\e[0m" - fi - fi - show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR diff --git a/init/modules/EESSI/2023.06.lua b/init/modules/EESSI/2023.06.lua index eb1cd1753b..d5105e89fc 100644 --- a/init/modules/EESSI/2023.06.lua +++ b/init/modules/EESSI/2023.06.lua @@ -49,15 +49,6 @@ function archdetect_cpu() -- We loop over the list, and return the highest matching arch for which a directory exists for this EESSI version for archdetect_filter_cpu in string.gmatch(archdetect_options, "([^" .. ":" .. "]+)") do if isDir(pathJoin(eessi_prefix, "software", eessi_os_type, archdetect_filter_cpu, "software")) then - -- use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected, - -- since optimized software installations for Zen4 are a work-in-progress, - -- see https://gitlab.com/eessi/support/-/issues/37 - if (archdetect_filter_cpu == "x86_64/amd/zen4" and not os.getenv("EESSI_SOFTWARE_SUBDIR_OVERRIDE") == "x86_64/amd/zen4") then - archdetect_filter_cpu = "x86_64/amd/zen3" - if mode() == "load" then - LmodMessage("Sticking to " .. archdetect_filter_cpu .. " for now, since optimized installations for AMD Genoa (Zen4) are a work in progress.") - end - end eessiDebug("Selected archdetect CPU: " .. archdetect_filter_cpu) return archdetect_filter_cpu end diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh index 192a36a483..e7997cad3d 100755 --- a/install_apptainer_ubuntu.sh +++ b/install_apptainer_ubuntu.sh @@ -2,30 +2,13 @@ set -e -# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 -sudo apt-get install alien -alien --version -epel_subdir="pub/epel/8" -apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') -curl -OL https://dl.fedoraproject.org/${epel_subdir}/Everything/x86_64/Packages/a/${apptainer_rpm} -sudo alien -d ${apptainer_rpm} -sudo apt install ./apptainer*.deb -# No unpriviledged user name spaces in Ubuntu 23.10+ -ubuntu_version=$(lsb_release -r | awk '{print $2}') -if [[ $(echo -e "$ubuntu_version\n23.10" | sort -V | head -n 1) == "23.10" ]]; then - sudo tee /etc/apparmor.d/apptainer << 'EOF' -# Permit unprivileged user namespace creation for apptainer starter -abi , -include -profile apptainer /usr/libexec/apptainer/bin/starter{,-suid} - flags=(unconfined) { - userns, - # Site-specific additions and overrides. See local/README for details. - include if exists -} -EOF - sudo systemctl reload apparmor -fi +sudo apt update +sudo apt install -y software-properties-common + +sudo add-apt-repository -y ppa:apptainer/ppa +sudo apt update +sudo apt install -y apptainer-suid + apptainer --version # also check whether 'singularity' command is still provided by Apptainer installation singularity --version diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl deleted file mode 100644 index 323aafd5ec..0000000000 --- a/reframe_config_bot.py.tmpl +++ /dev/null @@ -1,57 +0,0 @@ -# WARNING: this file is intended as template and the __X__ template variables need to be replaced -# before it can act as a configuration file -# Once replaced, this is a config file for running tests after the build phase, by the bot - -from eessi.testsuite.common_config import common_logging_config -from eessi.testsuite.constants import * # noqa: F403 - - -site_configuration = { - 'systems': [ - { - 'name': 'BotBuildTests', - 'descr': 'Software-layer bot', - 'hostnames': ['.*'], - 'modules_system': 'lmod', - 'partitions': [ - { - 'name': '__RFM_PARTITION__', - 'scheduler': 'local', - 'launcher': 'mpirun', - 'environs': ['default'], - 'features': [ - FEATURES[CPU] - ] + list(SCALES.keys()), - 'resources': [ - { - 'name': 'memory', - 'options': ['--mem={size}'], - } - ], - 'extras': { - # Make sure to round down, otherwise a job might ask for more mem than is available - # per node - 'mem_per_node': __MEM_PER_NODE__, - }, - 'max_jobs': 1 - } - ] - } - ], - 'environments': [ - { - 'name': 'default', - 'cc': 'cc', - 'cxx': '', - 'ftn': '' - } - ], - 'general': [ - { - 'purge_environment': True, - 'resolve_module_conflicts': False, # avoid loading the module before submitting the job - 'remote_detect': True, - } - ], - 'logging': common_logging_config(), -} diff --git a/run_tests.sh b/run_tests.sh index 1dbb47db9d..f6264c3cc8 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -17,8 +17,14 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults +# Make sure we clone the latest version. This assumes versions are of the format "v1.2.3", then picks the latest +# then checks it out +TEST_CLONE="git clone https://github.com/EESSI/test-suite EESSI-test-suite && cd EESSI-test-suite" +LATEST_VERSION="VERSION=\$(git tag | grep '^v[0-9]\+\.[0-9]\+\.[0-9]\+$' | sort -t. -k 1,1n -k 2,2n -k 3,3n | tail -1)" +CHECKOUT_LATEST="git checkout \${VERSION}" + # Git clone has to be run in compat layer, to make the git command available -./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite" +./run_in_compat_layer_env.sh "${TEST_CLONE} && ${LATEST_VERSION} && ${CHECKOUT_LATEST}" # Run the test suite ./test_suite.sh "$@" diff --git a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh index 3842aff307..954cf45cf7 100755 --- a/scripts/gpu_support/nvidia/install_cuda_host_injections.sh +++ b/scripts/gpu_support/nvidia/install_cuda_host_injections.sh @@ -147,9 +147,9 @@ else fi avail_space=$(df --output=avail "${tmpdir}"/ | tail -n 1 | awk '{print $1}') if (( avail_space < required_space_in_tmpdir )); then - error="Need at least ${required_space_in_tmpdir} disk space under ${tmpdir}.\n" - error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check." - error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH " + error="Need at least ${required_space_in_tmpdir}GB disk space under ${tmpdir}.\n" + error="${error}Set the environment variable CUDA_TEMP_DIR to a location with adequate space to pass this check.\n" + error="${error}You can alternatively set EASYBUILD_BUILDPATH and/or EASYBUILD_SOURCEPATH\n" error="${error}to reduce this requirement. Exiting now..." fatal_error "${error}" fi diff --git a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh index 2218a92116..27447341fd 100755 --- a/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +++ b/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh @@ -1,14 +1,72 @@ #!/bin/bash -# This script links host libraries related to GPU drivers to a location where -# they can be found by the EESSI linker (or sets LD_PRELOAD as an -# alternative.) - -# Initialise our bash functions -TOPDIR=$(dirname "$(realpath "$BASH_SOURCE")") +# NVIDIA Host Libraries Linking Script for EESSI +# ============================================ +# Overview: +# 1. Initialize environment and source utility functions +# All function definitions be here. +# 2. Check prerequisites: +# - EESSI environment initialization +# - nvidia-smi availability +# - Proper umask settings for global read permissions +# 3. Gather NVIDIA information: +# - Detect GPU driver version +# - Get CUDA version +# 4. Library detection and matching: +# - Download/use default NVIDIA library list +# - Find host libraries using ldconfig +# - Match required NVIDIA libraries +# 5. Handle two operation modes: +# a) Show LD_PRELOAD mode: Displays environment variables for preloading +# Suggest exports for following variables: +# EESSI_GPU_COMPAT_LD_PRELOAD (Minimal LD_PRELOAD) +# EESSI_GPU_LD_PRELOAD (Full LD_PRELOAD) +# EESSI_OVERRIDE_GPU_CHECK +# b) Symlink mode: Create directory structure and link libraries +# Create necessary symlinks in EESSI directory structure +# +# Error Handling: +# - nvidia-smi detection: Exits if NVIDIA drivers not found +# - Library matching: Reports missing libraries +# - Permission issues: Checks write access and umask settings +# - Symlink conflicts: Validates existing symlinks +# - Directory creation: Ensures proper structure exists +# +# Note: This script is part of EESSI (European Environment for Scientific +# Software Installations) and manages the linking of host NVIDIA libraries +# to make them accessible within the EESSI environment. + +# ###################################################### # +# 1. Initialize environment and source utility functions # +# ###################################################### # + +TOPDIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")") source "$TOPDIR"/../../utils.sh -# Define a function to find the host ld_config +# Command line help function +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " --show-ld-preload Enable recommendations for LD_PRELOAD mode" + echo " --no-download Don't download list of Nvidia libraries from URL," + echo " but use hardcoded list here in get_nvlib_list()" + echo " -v, --verbose Display debugging messages," + echo " actions taken, commands being run." +} + +# Initialize global variables (These are accessed or set from functions) +LD_PRELOAD_MODE=0 # Trigger show-ld-preload mode T/F +LIBS_LIST="" # Command line argument for get_nvlib_list +VERBOSE=0 # Set verbosity logging T/F +HOST_GPU_DRIVER_VERSION="" # GPU Driver version () +HOST_GPU_CUDA_VERSION="" # GPU CUDA version () +MATCHED_LIBRARIES=() # List of found CUDA libraries based on get_nvlib_list() +MISSING_LIBRARIES=() # Complementary to Matched libraries. + + +# Locates the host system's ldconfig, avoiding CVMFS paths +# Returns path to first valid ldconfig found, prioritizing /sbin get_host_ldconfig() { local command_name="ldconfig" # Set command to find local exclude_prefix="/cvmfs" # Set excluded prefix (paths to ignore) @@ -44,8 +102,17 @@ get_host_ldconfig() { fi } +# Downloads or provides default list of required NVIDIA libraries. +# As echo to stdout! Don't print any messages inside this function. +# Returns 0 if download successful, 1 if using default list get_nvlib_list() { local nvliblist_url="https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf" + + # see https://apptainer.org/docs/admin/1.0/configfiles.html#nvidia-gpus-cuda + # https://github.com/apptainer/apptainer/commits/main/etc/nvliblist.conf + # This default_nvlib_list is based on this commit on Oct 1, 2024: + # https://github.com/apptainer/apptainer/commit/a19fa01527a8914839b8d1649688f83c61ba9ad2 + # TODO: driver version which corresponds to? local default_nvlib_list=( "libcuda.so" "libcudadebugger.so" @@ -94,7 +161,7 @@ get_nvlib_list() { "tls_test_.so" ) - # Check if the function was called with the "default" argument + # Check if the function was called with the "default" argument if [[ "$1" == "default" ]]; then printf "%s\n" "${default_nvlib_list[@]}" return 1 @@ -106,124 +173,96 @@ get_nvlib_list() { # Check if curl failed (i.e., the content is empty) if [ -z "$nvliblist_content" ]; then # Failed to download nvliblist.conf, using default list instead + # We can't echo here + # echo_yellow "Download failed, using default list of libraries instead" printf "%s\n" "${default_nvlib_list[@]}" return 1 fi # If curl succeeded, filter and return the libraries from the downloaded content echo "$nvliblist_content" | grep '.so$' - + + # We can't echo here + # echo "Using downloaded list of libraries" return 0 } -# Function to check if umask allows global read +# Verifies if current umask allows global read access +# Exits with error if permissions are too restrictive check_global_read() { # Get the current umask value - local current_umask=$(umask) + local current_umask + current_umask=$(umask) + log_verbose "current umask: ${current_umask}" # Convert umask to decimal to analyze - local umask_octal=$(printf '%03o\n' "$current_umask") + local umask_octal + umask_octal=$(printf '%03o\n' "$current_umask") # Check if umask allows global read if [ "$umask_octal" -gt 022 ]; then fatal_error "The current umask ($current_umask) does not allow global read permissions, you'll want everyone to be able to read the created directory." fi + # TODO: Option to set $UMASK here? + # https://github.com/EESSI/software-layer/pull/754#discussion_r1950643598 } -# Make sure EESSI is initialised (doesn't matter what version) -check_eessi_initialised - -# Check for required commands -command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, this script won't do anything useful"; return 1; } - -# Variables -LD_PRELOAD_MODE=0 -LIBS_LIST="" - -# Parse command-line options -while [[ "$#" -gt 0 ]]; do - case "$1" in - --ld-preload) LD_PRELOAD_MODE=1 ;; # Enable LD_PRELOAD mode - --no-download) LIBS_LIST="default" ;; # Download latest list of CUDA libraries - *) fatal_error "Unknown option: $1";; - esac - shift -done - -# Gather information about NVIDIA drivers (even if we are inside a Gentoo Prefix in a container) -export LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}" - -# Check for NVIDIA GPUs via nvidia-smi command -nvidia_smi=$(command -v nvidia-smi) -if [[ $? -eq 0 ]]; then - nvidia_smi_out=$(mktemp -p /tmp nvidia_smi_out.XXXXX) - nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader 2>&1 > $nvidia_smi_out - if [[ $? -eq 0 ]]; then - nvidia_smi_info=$(head -1 "${nvidia_smi_out}") - host_cuda_version=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f4 -d,) - host_driver_version=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f3 -d,) - echo_green "Found host CUDA version ${host_cuda_version}" - echo_green "Found NVIDIA GPU driver version ${host_driver_version}" - rm -f $nvidia_smi_out +# Checks for nvidia-smi command and extracts GPU information +# Sets HOST_GPU_CUDA_VERSION and HOST_GPU_DRIVER_VERSION variables +check_nvidia_smi_info() { + + if command -v nvidia-smi + then + log_verbose "Found nvidia-smi at: $(which nvidia-smi)" + + # Create temporary file for nvidia-smi output + nvidia_smi_out=$(mktemp -p /tmp nvidia_smi_out.XXXXX) + log_verbose "Creating temporary output file: ${nvidia_smi_out}" + + # Query GPU information and parse versions + + if nvidia-smi --query-gpu=gpu_name,count,driver_version,compute_cap --format=csv,noheader > "$nvidia_smi_out" 2>&1 + then + nvidia_smi_info=$(head -1 "${nvidia_smi_out}") + HOST_GPU_CUDA_VERSION=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f4 -d,) + HOST_GPU_DRIVER_VERSION=$(echo "${nvidia_smi_info}" | sed 's/, /,/g' | cut -f3 -d,) + echo_green "Found host CUDA version ${HOST_GPU_CUDA_VERSION}" + echo_green "Found NVIDIA GPU driver version ${HOST_GPU_DRIVER_VERSION}" + rm -f "$nvidia_smi_out" + else + fatal_error "nvidia-smi command failed, see output in $nvidia_smi_out. Please remove the file afterwards." + fi else - fatal_error "nvidia-smi command failed, see output in $nvidia_smi_out" + fatal_error "nvidia-smi command not found" + exit 2 fi -else - fatal_error "nvidia-smi command not found" - exit 2 -fi - -# Gather any CUDA related driver libraries from the host -# - First let's see what driver libraries are there -# - then extract the ones we need for CUDA - -# Find the host ldconfig -host_ldconfig=$(get_host_ldconfig) -# Gather libraries on the host (_must_ be host ldconfig) -host_libraries=$("${host_ldconfig}" -p | awk '{print $NF}') -singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) - -# Now gather the list of possible CUDA libraries and make them into an array -cuda_candidate_libraries=($(get_nvlib_list "${LIBS_LIST}")) -# Check if the function returned an error (e.g., curl failed) -if [ $? -ne 0 ]; then - echo "Using default list of libraries" -else - echo "Using downloaded list of libraries" -fi - -# Filter the host libraries to find the CUDA libaries locations -# Initialize an array to hold the matched libraries -matched_libraries=() - -# Process each library and check for matches in libs.txt -for library in "${cuda_candidate_libraries[@]}"; do - # Search for the library in libs.txt and add it to the matched_libraries array - matched=$(echo "$host_libraries $singularity_libs" | grep "$library") - if [ -n "$matched" ]; then - matched_libraries+=( $matched ) # Add matched library to the array - fi -done - -# Output the number of matched libraries -echo "Matched ${#matched_libraries[@]} CUDA Libraries" +} -# LD_PRELOAD Mode -if [ "$LD_PRELOAD_MODE" -eq 1 ]; then +# Suggests configurations for LD_PRELOAD environment for CUDA libraries +# Filters libraries and configures both minimal and full preload options +show_ld_preload() { + echo echo_yellow "When attempting to use LD_PRELOAD we exclude anything related to graphics" + + # Define core CUDA libraries needed for compute cuda_compat_nvlib_list=( "libcuda.so" "libcudadebugger.so" "libnvidia-nvvm.so" "libnvidia-ptxjitcompiler.so" ) + # Filter out all symlinks and libraries that have missing library dependencies under EESSI filtered_libraries=() compat_filtered_libraries=() - for library in "${matched_libraries[@]}"; do + + for library in "${MATCHED_LIBRARIES[@]}"; do + # Run ldd on the given binary and filter for "not found" libraries - not_found_libs=$(ldd "${library}" 2>/dev/null | grep "not found" | awk '{print $1}') + # not_found_libs=$(ldd "${library}" 2>/dev/null | grep "not found" | awk '{print $1}') + # Trim multiple spaces then use cut + not_found_libs=$(ldd "${library}" 2>/dev/null | grep "not found" | tr -s ' ' | cut -d' ' -f1) # Check if it is missing an so dep under EESSI if [[ -z "$not_found_libs" ]]; then # Resolve any symlink @@ -247,7 +286,7 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then all_found=true for lib in $not_found_libs; do found=false - for listed_lib in "${matched_libraries[@]}"; do + for listed_lib in "${MATCHED_LIBRARIES[@]}"; do # Matching to the .so or a symlink target is enough realpath_lib=$(realpath "${listed_lib}") if [[ "$lib" == "$listed_lib"* || "$realpath_lib" == *"$lib" ]]; then @@ -287,130 +326,366 @@ if [ "$LD_PRELOAD_MODE" -eq 1 ]; then # Set EESSI_GPU_LD_PRELOAD with the matched libraries if [ ${#filtered_libraries[@]} -gt 0 ]; then - echo - echo_yellow "The recommended way to use LD_PRELOAD is to only use it when you need to." - echo - EESSI_GPU_COMPAT_LD_PRELOAD=$(printf "%s\n" "${compat_filtered_libraries[@]}" | tr '\n' ':') - # Remove the trailing colon from LD_PRELOAD if it exists - EESSI_GPU_COMPAT_LD_PRELOAD=${EESSI_GPU_COMPAT_LD_PRELOAD%:} - export EESSI_GPU_COMPAT_LD_PRELOAD - echo_yellow "A minimal preload which should work in most cases:" - echo_green "export EESSI_GPU_COMPAT_LD_PRELOAD=\"$EESSI_GPU_COMPAT_LD_PRELOAD\"" - echo - - EESSI_GPU_LD_PRELOAD=$(printf "%s\n" "${filtered_libraries[@]}" | tr '\n' ':') - # Remove the trailing colon from LD_PRELOAD if it exists - EESSI_GPU_LD_PRELOAD=${EESSI_GPU_LD_PRELOAD%:} - export EESSI_GPU_LD_PRELOAD - echo_yellow "A corner-case full preload (which is hard on memory) for exceptional use:" - - echo_green "export EESSI_GPU_LD_PRELOAD=\"$EESSI_GPU_LD_PRELOAD\"" - export EESSI_OVERRIDE_GPU_CHECK=1 - echo_green "export EESSI_OVERRIDE_GPU_CHECK=\"$EESSI_OVERRIDE_GPU_CHECK\"" - echo - echo_yellow "Then you can set LD_PRELOAD only when you want to run a GPU application, e.g.," - echo_yellow " LD_PRELOAD=\"\$EESSI_GPU_COMPAT_LD_PRELOAD\" device_query" + echo + echo_yellow "The recommended way to use LD_PRELOAD is to only use it when you need to." + echo + + # Set up MINIMAL preload for common cases + EESSI_GPU_COMPAT_LD_PRELOAD=$(printf "%s\n" "${compat_filtered_libraries[@]}" | tr '\n' ':') + # Remove the trailing colon from LD_PRELOAD if it exists + EESSI_GPU_COMPAT_LD_PRELOAD=${EESSI_GPU_COMPAT_LD_PRELOAD%:} + export EESSI_GPU_COMPAT_LD_PRELOAD + + echo_yellow "A minimal preload which should work in most cases:" + echo_green "export EESSI_GPU_COMPAT_LD_PRELOAD=\"$EESSI_GPU_COMPAT_LD_PRELOAD\"" + echo + + # Set up FULL preload for corner cases + EESSI_GPU_LD_PRELOAD=$(printf "%s\n" "${filtered_libraries[@]}" | tr '\n' ':') + # Remove the trailing colon from LD_PRELOAD if it exists + EESSI_GPU_LD_PRELOAD=${EESSI_GPU_LD_PRELOAD%:} + export EESSI_GPU_LD_PRELOAD + export EESSI_OVERRIDE_GPU_CHECK=1 + + echo_yellow "A corner-case full preload (which is hard on memory) for exceptional use:" + + # Display usage instructions + echo_green "export EESSI_GPU_LD_PRELOAD=\"$EESSI_GPU_LD_PRELOAD\"" + echo_green "export EESSI_OVERRIDE_GPU_CHECK=\"$EESSI_OVERRIDE_GPU_CHECK\"" + echo + echo_yellow "Then you can set LD_PRELOAD only when you want to run a GPU application," + echo_yellow "e.g. deviceQuery command from CUDA-Samples module:" + echo_yellow " LD_PRELOAD=\"\$EESSI_GPU_COMPAT_LD_PRELOAD\" deviceQuery" + echo_yellow "or LD_PRELOAD=\"\$EESSI_GPU_LD_PRELOAD\" deviceQuery" else - echo "No libraries matched, LD_PRELOAD not set." + echo "No libraries matched, LD_PRELOAD not set." fi - [[ "${BASH_SOURCE[0]}" != "${0}" ]] && return 1 || exit 1 -fi + [[ "${BASH_SOURCE[0]}" != "${0}" ]] && return 1 +} -# If we haven't already exited, we may need to create the symlinks +# Check host's ldconfig, gathers library paths, and filters them on matching. +# Sets MATCHED_LIBRARIES and MISSING_LIBRARIES +find_cuda_libraries_on_host() { + # First let's see what driver libraries are there + # then extract the ones we need for CUDA + + # Find the host ldconfig + host_ldconfig=$(get_host_ldconfig) + log_verbose "Found host ldconfig: ${host_ldconfig}" + + # Gather all libraries on the host (_must_ be host ldconfig). + # host_libraries=$("${host_ldconfig}" -p | awk '{print $NF}') + # Trim multiple spaces then use cut + host_libraries=$("${host_ldconfig}" -p | tr -s ' ' | cut -d' ' -f4) + # This is only for the scenario where the script is being run inside a container, if it fails the list is empty. + singularity_libs=$(ls /.singularity.d/libs/* 2>/dev/null) + + # Now gather the list of possible CUDA libraries and make them into an array + # https://www.shellcheck.net/wiki/SC2207 + cuda_candidate_libraries=($(get_nvlib_list "${LIBS_LIST}")) + # Check if the function returned an error (e.g., curl failed) + # Echo here, we take stdout from function as list of libraries. + if [ $? -ne 0 ]; then + echo "Using default list of libraries" + else + echo "Using downloaded list of libraries" + fi -# First let's make sure the driver libraries are not already in place -link_drivers=1 + # Search for CUDA Libraries in system paths + echo "Searching for CUDA Libraries" + for library in "${cuda_candidate_libraries[@]}"; do + + # Match libraries for current CPU architecture + # "contains" matching - (eg. 'libcuda.so' matches both 'libcuda.so' and 'libcuda.so.1') + # The `grep -v "i386"` is done to exclude i386 libraries, which could be installed in parallel with 64 libs. + matched=$(echo "$host_libraries $singularity_libs" | grep -v "i386" | grep "$library") + + if [ -n "$matched" ]; then + log_verbose "Found matches for ${library}: $matched" + + # Process each matched library and avoid duplicates by filename + # Used `while - read <<< $matched`` to handle whitespaces and special characters. + while IFS= read -r lib_path; do + # Skip empty lines + [ -z "$lib_path" ] && continue + + # Extract just the filename from the path + lib_name=$(basename "$lib_path") + echo "Checking library $lib_name for duplicates" + + # Check if we already have this library filename in our matched libraries + duplicate_found=0 + for existing_lib in "${MATCHED_LIBRARIES[@]}"; do + existing_name=$(basename "$existing_lib") + if [ "$existing_name" = "$lib_name" ]; then + log_verbose "Duplicate library found: $lib_name (existing: $existing_lib, currently processed: $lib_path)" + log_verbose "Discarting $lib_path" + duplicate_found=1 + break + fi + done + + # If no duplicate found, add this library + if [ "$duplicate_found" -eq 0 ]; then + MATCHED_LIBRARIES+=("$lib_path") + fi + done <<< "$matched" + else + # There are some libraries, that weren't matched/found on the system + log_verbose "No matches found for ${library}" + MISSING_LIBRARIES+=("$library") + fi + done -# Make sure that target of host_injections variant symlink is an existing directory -host_injections_target=$(realpath -m "${EESSI_CVMFS_REPO}/host_injections") -if [ ! -d "$host_injections_target" ]; then - check_global_read - create_directory_structure "$host_injections_target" -fi -host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" -host_injection_driver_dir="${host_injections_nvidia_dir}/host" -host_injection_driver_version_file="${host_injection_driver_dir}/driver_version.txt" -if [ -e "$host_injection_driver_version_file" ]; then - if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then - echo_green "The host GPU driver libraries (v${host_driver_version}) have already been linked! (based on ${host_injection_driver_version_file})" - link_drivers=0 - else - # There's something there but it is out of date - echo_yellow "Cleaning out outdated symlinks" - rm "${host_injection_driver_dir}"/* || fatal_error "Unable to remove files under '${host_injection_driver_dir}'." - fi -fi + # Report matching results + echo_green "Matched ${#MATCHED_LIBRARIES[@]} CUDA Libraries" -drivers_linked=0 -if [ "$link_drivers" -eq 1 ]; then - check_global_read - if ! create_directory_structure "${host_injection_driver_dir}" ; then - fatal_error "No write permissions to directory ${host_injection_driver_dir}" - fi - cd "${host_injection_driver_dir}" || fatal_error "Failed to cd to ${host_injection_driver_dir}" - - # Make symlinks to all the interesting libraries - # Loop over each matched library - for library in "${matched_libraries[@]}"; do - # Create a symlink in the current directory - ln -s "$library" . - # Check if the symlink was created successfully - if [ $? -ne 0 ]; then - fatal_error "Error: Failed to create symlink for library $library in $PWD" - fi - done - - # Inject driver and CUDA versions into the directory - echo "$host_driver_version" > driver_version.txt - echo "$host_cuda_version" > cuda_version.txt - drivers_linked=1 -fi + if [ ${#MISSING_LIBRARIES[@]} -gt 0 ]; then + echo_yellow "The following libraries were not found (based on 'get_nvlib_list')" + printf '%s\n' "${MISSING_LIBRARIES[@]}" + fi +} -# Make latest symlink for NVIDIA drivers -cd "$host_injections_nvidia_dir" || fatal_error "Failed to cd to $host_injections_nvidia_dir" -symlink="latest" -if [ -L "$symlink" ]; then - if [ "$drivers_linked" -eq 1 ]; then - ln -sf host "$symlink" - if [ $? -eq 0 ]; then +# Actually symlinks the Matched libraries to correct folders. +# Then also creates "host" and "latest" folder symlinks +symlink_mode () { + # First let's make sure the driver libraries are not already in place + # Have to link drivers = True + link_drivers=1 + + # Make sure that target of host_injections variant symlink is an existing directory + echo "Ensure host_injections directory" + host_injections_target=$(realpath -m "${EESSI_CVMFS_REPO}/host_injections") + log_verbose "host_injections_target: ${host_injections_target}" + if [ ! -d "$host_injections_target" ]; then + check_global_read + create_directory_structure "$host_injections_target" + fi + + # Define proper nvidia directory structure for host_injections in EESSI + host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}" + host_injection_driver_dir="${host_injections_nvidia_dir}/host" + host_injection_driver_version_file="${host_injection_driver_dir}/driver_version.txt" + log_verbose "host_injections_nvidia_dir: ${host_injections_nvidia_dir}" + log_verbose "host_injection_driver_dir: ${host_injection_driver_dir}" + log_verbose "host_injection_driver_version_file: ${host_injection_driver_version_file}" + + # Check if drivers are already linked with correct version + # This is done by comparing host_injection_driver_version_file (driver_version.txt) + # This is needed when updating GPU drivers. + if [ -e "$host_injection_driver_version_file" ]; then + if grep -q "$HOST_GPU_DRIVER_VERSION" "$host_injection_driver_version_file"; then + echo_green "The host GPU driver libraries (v${HOST_GPU_DRIVER_VERSION}) have already been linked! (based on ${host_injection_driver_version_file})" + # The GPU libraries were already linked for this version of CUDA driver + # Have to link drivers = False + link_drivers=0 + else + # There's something there but it is out of date + echo_yellow "The host GPU driver libraries version have changed. Now its: (v${HOST_GPU_DRIVER_VERSION})" + echo_yellow "Cleaning out outdated symlinks." + rm "${host_injection_driver_dir}"/* || fatal_error "Unable to remove files under '${host_injection_driver_dir}'." + fi + fi + + # Link all matched_libraries from Nvidia to correct host_injection folder + # This step is only run, when linking of drivers is needed (eg. link_drivers==1) + # Setup variable to track if some drivers were actually linked this run. + drivers_linked=0 + + # Have to link drivers + if [ "$link_drivers" -eq 1 ]; then + # Link the matched libraries + + echo_green "Linking drivers to the host_injection folder" + check_global_read + if ! create_directory_structure "${host_injection_driver_dir}" ; then + fatal_error "No write permissions to directory ${host_injection_driver_dir}" + fi + + cd "${host_injection_driver_dir}" || fatal_error "Failed to cd to ${host_injection_driver_dir}" + log_verbose "Changed directory to: $PWD" + + # Make symlinks to all the interesting libraries + # Loop over each matched library + for library in "${MATCHED_LIBRARIES[@]}"; do + log_verbose "Linking library: ${library}" + + # Get just the library filename + lib_name=$(basename "$library") + + # Check if the symlink already exists + if [ -L "$lib_name" ]; then + # Check if it's pointing to the same target + target=$(readlink "$lib_name") + if [ "$target" = "$library" ]; then + log_verbose "Symlink for $lib_name already exists and points to correct target" + continue + else + log_verbose "Symlink for $lib_name exists but points to wrong target: $target, updating..." + rm "$lib_name" + fi + fi + + # Create a symlink in the current directory + # and check if the symlink was created successfully + if ! ln -s "$library" . + then + fatal_error "Error: Failed to create symlink for library $library in $PWD" + fi + done + + # Inject driver and CUDA versions into the directory + echo "$HOST_GPU_DRIVER_VERSION" > driver_version.txt + echo "$HOST_GPU_CUDA_VERSION" > cuda_version.txt + + drivers_linked=1 + fi + + # Make latest symlink for NVIDIA drivers + cd "$host_injections_nvidia_dir" || fatal_error "Failed to cd to $host_injections_nvidia_dir" + log_verbose "Changed directory to: $PWD" + symlink="latest" + + # Check if the symlink exists + if [ -L "$symlink" ]; then + # If the drivers were linked this run - relink the symlink! + if [ "$drivers_linked" -eq 1 ]; then + # Force relinking the current link. + # Need to remove the link first, otherwise this will follow existing symlink + # and create host directory one level down ! + rm "$symlink" || fatal_error "Failed to remove symlink ${symlink}" + + if ln -sf host "$symlink" + then + echo "Successfully force recreated symlink between $symlink and host in $PWD" + else + fatal_error "Failed to force recreate symlink between $symlink and host in $PWD" + fi + fi + else + # If the symlink doesn't exists, create normal one. + if ln -s host "$symlink" + then echo "Successfully created symlink between $symlink and host in $PWD" else fatal_error "Failed to create symlink between $symlink and host in $PWD" fi fi -else - ln -s host "$symlink" - if [ $? -eq 0 ]; then - echo "Successfully created symlink between $symlink and host in $PWD" + + # Make sure the libraries can be found by the EESSI linker + host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections} + if [ -L "$host_injection_linker_dir/lib" ]; then + # Use readlink without -f to get direct symlink target + # using -f option will create "lastest" symlink one dir deeper (inside host) + target_path=$(readlink "$host_injection_linker_dir/lib") + expected_target="$host_injections_nvidia_dir/latest" + + log_verbose "Checking symlink target for EESSI linker:" + log_verbose "Current target: $target_path" + log_verbose "Expected target: $expected_target" + + # Update symlink if needed + if [ "$target_path" != "$expected_target" ]; then + cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" + log_verbose "Changed directory to: $PWD" + + + if ln -sf "$expected_target" lib + then + echo "Successfully force created symlink between $expected_target and lib in $PWD" + else + fatal_error "Failed to force create symlink between $expected_target and lib in $PWD" + fi + else + log_verbose "Symlink already points to correct target" + fi else - fatal_error "Failed to create symlink between $symlink and host in $PWD" + # Just start from scratch, symlink doesn't exists. + check_global_read + create_directory_structure "$host_injection_linker_dir" + cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" + log_verbose "Changed directory to: $PWD" + + if ln -s "$host_injections_nvidia_dir/latest" lib + then + echo "Successfully created symlink between $host_injections_nvidia_dir/latest and lib in $PWD" + else + fatal_error "Failed to create symlink between $host_injections_nvidia_dir/latest and lib in $PWD" + fi fi -fi -# Make sure the libraries can be found by the EESSI linker -host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections} -if [ -L "$host_injection_linker_dir/lib" ]; then - target_path=$(readlink -f "$host_injection_linker_dir/lib") - if [ "$target_path" != "$host_injections_nvidia_dir/latest" ]; then - cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" - ln -sf "$host_injections_nvidia_dir/latest" lib - if [ $? -eq 0 ]; then - echo "Successfully created symlink between $host_injections_nvidia_dir/latest and lib in $PWD" - else - fatal_error "Failed to create symlink between $host_injections_nvidia_dir/latest and lib in $PWD" +} + +# Logging function for verbose mode +# TODO: move to utils? +log_verbose() { + if [ "$VERBOSE" -eq 1 ]; then + echo "[VERBOSE] $*" fi - fi -else - check_global_read - create_directory_structure "$host_injection_linker_dir" - cd "$host_injection_linker_dir" || fatal_error "Failed to cd to $host_injection_linker_dir" - ln -s "$host_injections_nvidia_dir/latest" lib - if [ $? -eq 0 ]; then - echo "Successfully created symlink between $host_injections_nvidia_dir/latest and lib in $PWD" - else - fatal_error "Failed to create symlink between $host_injections_nvidia_dir/latest and lib in $PWD" - fi +} + + +# ############################################### +# 2. Check prerequisites # +# ############################################### + +# Make sure EESSI is initialised (doesn't matter what version) +check_eessi_initialised + +# Verify nvidia-smi availability +log_verbose "Checking for nvidia-smi command..." +command -v nvidia-smi >/dev/null 2>&1 || { echo_yellow "nvidia-smi not found, this script won't do anything useful"; return 1; } + +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; # Show help + --show-ld-preload) LD_PRELOAD_MODE=1 ;; # Enable LD_PRELOAD mode + --no-download) LIBS_LIST="default" ;; # Download latest list of CUDA libraries + --verbose|-v) VERBOSE=1 ;; # Enable verbose output + *) + show_help + fatal_error "Unknown option: $1" + ;; + esac + shift +done + +# ############################################### +# 3. Gather NVIDIA information # +# ############################################### + +# Gather information about NVIDIA drivers (even if we are inside a Gentoo Prefix in a container) +export LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}" + +# Check for NVIDIA GPUs via nvidia-smi command +check_nvidia_smi_info + +# ############################################### +# 4. Library detection and matching # +# ############################################### + +# Gather any CUDA related driver libraries from the host +# Sets MATCHED_LIBRARIES and MISSING_LIBRARIES array variables +find_cuda_libraries_on_host + +# ############################################### +# 5. Handle operation modes # +# ############################################### + +# === 5a. LD_PRELOAD Mode === +if [ "$LD_PRELOAD_MODE" -eq 1 ]; then + show_ld_preload + exit 0 fi +# === 5b. Symlink Mode === +# If we haven't already exited, we may need to create the symlinks +symlink_mode + +# If everything went OK, show success message echo_green "Host NVIDIA GPU drivers linked successfully for EESSI" diff --git a/test_suite.sh b/test_suite.sh index 1f0b91c477..f5f3255841 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -132,56 +132,42 @@ else fi # Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration -export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py -export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl +# RFM_CONFIG_FILES _has_ to be set by the site hosting the bot, so that it knows where to find the ReFrame +# config file that matches the bot config. See https://gitlab.com/eessi/support/-/issues/114#note_2293660921 +if [ -z "$RFM_CONFIG_FILES" ]; then + if [ -z "${shared_fs_path}" ]; then + fatal_error "Environment variable 'shared_fs_path' was expected, but was not set" + fi + # Try to find a config file at $shared_fs_path/reframe_config.py + export RFM_CONFIG_FILES="${shared_fs_path}/reframe_config.py" + if [ ! -f "${RFM_CONFIG_FILES}" ]; then + # If we haven't found the ReFrame config, print an informative error + err_msg="Please put a ReFrame configuration file in ${shared_fs_path}/reframe_config.py" + err_msg="${err_msg} or set RFM_CONFIG_FILES in the environment of this bot instance to point to a valid" + err_msg="${err_msg} ReFrame configuration file that matches the bot config." + err_msg="${err_msg} For more information, see https://gitlab.com/eessi/support/-/issues/114#note_2293660921" + fatal_error "${err_msg}" + fi +fi export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests export RFM_CHECK_SEARCH_RECURSIVE=1 export RFM_PREFIX=$PWD/reframe_runs -echo "Configured reframe with the following environment variables:" -env | grep "RFM_" - -# The /sys inside the container is not the same as the /sys of the host -# We want to extract the memory limit from the cgroup on the host (which is typically set by SLURM). -# Thus, bot/test.sh bind-mounts the host's /sys/fs/cgroup into /hostsys/fs/cgroup -# and that's the prefix we use to extract the memory limit from -cgroup_v1_mem_limit="/hostsys/fs/cgroup/memory/$( "/tmp/nvidia_libs/$lib" + # Make it executable to pass potential file type checks + chmod +x "/tmp/nvidia_libs/$lib" + # Create a symlink for libraries without version number (if the library has one) + if [[ "$lib" == *".so."* ]]; then + base_lib=$(echo "$lib" | sed 's/\.so\.[0-9]*/.so/') + ln -sf "/tmp/nvidia_libs/$lib" "/tmp/nvidia_libs/$base_lib" + fi + + # Create duplicate libraries in a different location + if [[ "$lib" == "libcuda.so.1" || "$lib" == "libnvidia-ml.so.1" ]]; then + echo "This is a duplicate $lib for testing purposes" > "/tmp/nvidia_libs_duplicate/$lib" + chmod +x "/tmp/nvidia_libs_duplicate/$lib" + if [[ "$lib" == *".so."* ]]; then + base_lib=$(echo "$lib" | sed 's/\.so\.[0-9]*/.so/') + ln -sf "/tmp/nvidia_libs_duplicate/$lib" "/tmp/nvidia_libs_duplicate/$base_lib" + fi + fi +done + +# Create a fake ldconfig cache that points to our fake libraries +mkdir -p /tmp/ldconfig + +# Create a wrapper script for ldconfig +cat > /tmp/ldconfig/ldconfig << 'EOF' +#!/bin/bash +# Fake ldconfig command that returns our fake libraries + +if [ "$1" = "-p" ]; then + # Simulate ldconfig -p output with duplicate entries + echo "libcuda.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libcuda.so.1" + echo "libcuda.so.1 (libc6,x86-64) => /tmp/nvidia_libs_duplicate/libcuda.so.1" + echo "libcuda.so (libc6,x86-64) => /tmp/nvidia_libs/libcuda.so" + echo "libcuda.so (libc6,x86-64) => /tmp/nvidia_libs_duplicate/libcuda.so" + echo "libnvidia-ml.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-ml.so.1" + echo "libnvidia-ml.so.1 (libc6,x86-64) => /tmp/nvidia_libs_duplicate/libnvidia-ml.so.1" + echo "libnvidia-ml.so (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-ml.so" + echo "libnvidia-ml.so (libc6,x86-64) => /tmp/nvidia_libs_duplicate/libnvidia-ml.so" + echo "libnvidia-ptxjitcompiler.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-ptxjitcompiler.so.1" + echo "libnvidia-ptxjitcompiler.so (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-ptxjitcompiler.so" + echo "libOpenCL.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libOpenCL.so.1" + echo "libOpenCL.so (libc6,x86-64) => /tmp/nvidia_libs/libOpenCL.so" + echo "libnvidia-fatbinaryloader.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-fatbinaryloader.so.1" + echo "libnvidia-fatbinaryloader.so (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-fatbinaryloader.so" + echo "libnvidia-opencl.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-opencl.so.1" + echo "libnvidia-opencl.so (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-opencl.so" + echo "libcudadebugger.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libcudadebugger.so.1" + echo "libcudadebugger.so (libc6,x86-64) => /tmp/nvidia_libs/libcudadebugger.so" + echo "libnvidia-compiler.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-compiler.so.1" + echo "libnvidia-compiler.so (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-compiler.so" + echo "libnvidia-nvvm.so.1 (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-nvvm.so.1" + echo "libnvidia-nvvm.so (libc6,x86-64) => /tmp/nvidia_libs/libnvidia-nvvm.so" +fi +EOF +chmod +x /tmp/ldconfig/ldconfig + +# Create a wrapper for ldd that returns minimal dependency info for our libraries +cat > /tmp/ldconfig/ldd << 'EOF' +#!/bin/bash +# Fake ldd command that doesn't show any missing dependencies +echo "linux-vdso.so.1 => (0x00007ffca39ed000)" +echo "libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f2ace77e000)" +echo "/lib64/ld-linux-x86-64.so.2 (0x00007f2aceb41000)" +EOF +chmod +x /tmp/ldconfig/ldd diff --git a/tests/nvidia-libs/mock-nvidia-smi.sh b/tests/nvidia-libs/mock-nvidia-smi.sh new file mode 100644 index 0000000000..313e71e2bf --- /dev/null +++ b/tests/nvidia-libs/mock-nvidia-smi.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# This is a standard fake nvidia-smi script for testing purposes +# Using driver version 535.129.03 and CUDA version 8.0 + +# Check if --query-gpu flag is used +if [[ "$1" == "--query-gpu=gpu_name,count,driver_version,compute_cap" && "$2" == "--format=csv,noheader" ]]; then + # Simulate output for a system with an NVIDIA A100 GPU + echo "NVIDIA A100, 1, 535.129.03, 8.0" + exit 0 +else + # Default output (similar to nvidia-smi with no arguments) + cat << EOF +Mon Feb 26 10:30:45 2024 ++-----------------------------------------------------------------------------+ +| NVIDIA-SMI 535.129.03 Driver Version: 535.129.03 CUDA Version: 12.2 | +|-------------------------------+----------------------+----------------------+ +| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC | +| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. | +| | | MIG M. | +|===============================+======================+======================| +| 0 NVIDIA A100 On | 00000000:00:00.0 Off | 0 | +| N/A 34C P0 69W / 400W | 0MiB / 40960MiB | 0% Default | +| | | Disabled | ++-------------------------------+----------------------+----------------------+ + ++-----------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=============================================================================| +| No running processes found | ++-----------------------------------------------------------------------------+ +EOF + exit 0 +fi From bd78807a6caa45456f860ffdbb29f50675c896ff Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 24 Apr 2025 17:34:05 +0200 Subject: [PATCH 109/112] Add comment on cvmfs_repo_name and remove blank line at EOF Co-authored-by: ocaisa --- create_tarball.sh | 3 +++ eessi_container.sh | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 6115865367..bae66fb836 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -26,6 +26,9 @@ if [ ! -d ${software_dir} ]; then exit 2 fi +# Need to extract the cvmfs_repo_name from the cvmfs_repo variable +# - remove /${EESSI_DEV_PROJECT} from the end (if it exists) +# - remove /cvmfs/ from the beginning cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" diff --git a/eessi_container.sh b/eessi_container.sh index 3edddf6c46..906eb2cbc2 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -769,7 +769,6 @@ do # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} - # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) if [[ ${cvmfs_repo_access} == "ro" ]] ; then # need to distinguish between basic "ro" access and "ro" after a "rw" session From 00d2e6b56b0daea68c5620bd326a43115a041163 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 24 Apr 2025 17:43:27 +0200 Subject: [PATCH 110/112] Comments on how $EESSI_DEV_PROJECT is handled --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index c0d9b03bc5..2bba0cba55 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -147,6 +147,8 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env +# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we +# append the project subdirectory to ${EESSI_CVMFS_REPO_OVERRIDE} export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" @@ -318,6 +320,10 @@ fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now source $software_layer_dir/init/eessi_defaults +# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we +# append the project (subdirectory) name to the end tarball name. This is information +# then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is +# appended export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) # Export EESSI_DEV_PROJECT to use it (if needed) when making tarball From 5c3154b5415055235fba7141fc6ecd5df1854833 Mon Sep 17 00:00:00 2001 From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com> Date: Thu, 24 Apr 2025 17:44:48 +0200 Subject: [PATCH 111/112] Improve comment on how SitePackage.lua and .lmodrc are handled Co-authored-by: ocaisa --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4d303906bb..5a3002d498 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -271,7 +271,7 @@ export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_S # If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then - # We keep track of the old install path so the SitePackage.lua and .lmodrc later + # We keep track of the old install path for settings paths to SitePackage.lua and .lmodrc later EASYBUILD_INSTALLPATH_STANDARD=${EASYBUILD_INSTALLPATH} # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_PROJECT_INSTALL unset EESSI_CVMFS_INSTALL From 6af5e85bf1d620784aaa3b221c13e7c655f08b9d Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 24 Apr 2025 17:48:24 +0200 Subject: [PATCH 112/112] Remove leftover lines These create ${EESSI_PROJECT_INSTALL} and export ${EESSI_DEV_PROJECT}. The first is handled inside the container, the latter must be exported already if this block is running --- EESSI-install-software.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4d303906bb..7e1d99da97 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -277,10 +277,6 @@ if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then unset EESSI_CVMFS_INSTALL export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" - # Consider removing this (created in eessi_container.sh) - mkdir -p ${EESSI_PROJECT_INSTALL} - # Need to export EESSI_DEV_PROJECT for eessi_container.sh - export ${EESSI_DEV_PROJECT} fi echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"