diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 713fcba883..099050b276 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -242,8 +242,8 @@ fi # order is important: these are needed to install a full CUDA SDK in host_injections # for now, this just reinstalls all scripts. Note the most elegant, but works -# Only run install_scripts.sh if not dev.eessi.io for security -if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then +# Only run install_scripts.sh if not in dev.eessi.io for security +if [[ -z ${EESSI_DEV_PROJECT} ]]; then ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX} fi @@ -268,8 +268,19 @@ export EESSI_CVMFS_INSTALL=1 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the # EESSI-extend module itself needs to be installed. export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + +# If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + # We keep track of the old install path for settings paths to SitePackage.lua and .lmodrc later + EASYBUILD_INSTALLPATH_STANDARD=${EASYBUILD_INSTALLPATH} + # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_PROJECT_INSTALL + unset EESSI_CVMFS_INSTALL + export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE} + echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}" +fi + echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" -source load_eessi_extend_module.sh ${EESSI_VERSION} +source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION} echo "DEBUG: after loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" # Install full CUDA SDK and cu* libraries in host_injections @@ -319,11 +330,6 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then fi fi -# If in dev.eessi.io, allow building on top of software.eessi.io -if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then - module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all -fi - echo "DEBUG: adding path '$EASYBUILD_INSTALLPATH/modules/all' to MODULEPATH='${MODULEPATH}'" #module use $EASYBUILD_INSTALLPATH/modules/all safe_module_use $EASYBUILD_INSTALLPATH/modules/all @@ -396,7 +402,15 @@ else fi echo "DEBUG: before creating/updating lmod config files // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'" -export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" +if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then + # Make sure .lmod files are not checked for dev.eeessi.io + export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" + export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod" +else + export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" + export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" +fi + lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" echo "DEBUG: lmod_rc_file='${lmod_rc_file}'" if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then @@ -411,7 +425,6 @@ if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file" fi -export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_sitepackage_file diff --git a/bot/build.sh b/bot/build.sh index 2b35c63551..2bba0cba55 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -147,7 +147,9 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME} +# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we +# append the project subdirectory to ${EESSI_CVMFS_REPO_OVERRIDE} +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE} @@ -318,7 +320,15 @@ fi timestamp=$(date +%s) # to set EESSI_VERSION we need to source init/eessi_defaults now source $software_layer_dir/init/eessi_defaults -export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) +# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we +# append the project (subdirectory) name to the end tarball name. This is information +# then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is +# appended +export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp}) + +# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball +echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'" +export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT} # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh diff --git a/bot/inspect.sh b/bot/inspect.sh index 533968bffc..6ffd227b59 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -209,7 +209,7 @@ echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/$(cfg_get_value 'repository' 'repo_name')" +export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/$(cfg_get_value 'repository' 'repo_name')${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" echo "bot/inspect.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/create_tarball.sh b/create_tarball.sh index 01f498e1ac..bae66fb836 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -20,15 +20,18 @@ echo ">> tmpdir: $tmpdir" os="linux" source ${base_dir}/init/eessi_defaults cvmfs_repo=${EESSI_CVMFS_REPO} - software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir} ]; then echo "Software directory ${software_dir} does not exist?!" >&2 exit 2 fi -cvmfs_repo_name=${cvmfs_repo#/cvmfs/} -overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper" +# Need to extract the cvmfs_repo_name from the cvmfs_repo variable +# - remove /${EESSI_DEV_PROJECT} from the end (if it exists) +# - remove /cvmfs/ from the beginning +cvmfs_repo_name=${cvmfs_repo%"/${EESSI_DEV_PROJECT}"} +cvmfs_repo_name=${cvmfs_repo_name#/cvmfs/} +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then diff --git a/eessi_container.sh b/eessi_container.sh index d96ad041a8..906eb2cbc2 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -763,6 +763,8 @@ do cfg_repo_id=${cvmfs_repo_name} cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") fi + # remove project subdir in container + cvmfs_repo_name=${cvmfs_repo_name%"/${EESSI_DEV_PROJECT}"} # always create a directory for the repository (e.g., to store settings, ...) mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} @@ -812,8 +814,9 @@ do fi elif [[ ${cvmfs_repo_access} == "rw" ]] ; then # use repo-specific overlay directories - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT} + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" # set environment variables for fuse mounts in Singularity container diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index b4299c7a0d..e9a49cbe3d 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -35,6 +35,9 @@ fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_DEV_PROJECT} ]; then + INPUT="export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}; ${INPUT}" +fi if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}" fi