From 55066c3967b462f587e1135046c54d723386770b Mon Sep 17 00:00:00 2001 From: Adam McCartney Date: Thu, 24 Apr 2025 08:45:22 +0200 Subject: [PATCH 1/3] {2023.06}[NVHPC/25.1-CUDA-12.6] add hook for nvhpc This adds a pre_configure_hook for NVHPC. It performs some search and replace operations on the "localrc" file used by NVHPC to detect information about the system. In particular it points the sysroot flag at the eessi eprefix variable, and appends two variables definitions about where to look for system libraries. The content of the hook is extracted from: https://github.com/ComputeCanada/easybuild-computecanada-config/blob/main/2023/cc_hooks.py#L544-L547 --- eb_hooks.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 5f5405c173..d08a2b1608 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -744,6 +744,26 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def pre_configure_hook_nvhpc_x86_64(self, *args, **kwargs): + """ + pre-configure hook for nvhpc + - search and replace operations in the ec dict + """ + if self.name == "NVHPC": + eprefix = get_eessi_envvar("EPREFIX") + new_opts = f'''installdir=%(installdir)s/Linux_x86_64/%(version)s + EPREFIX={eprefix} + EBROOTGENTOO={eprefix}/usr + sed -i "s@\(set LDSO=.*\);@\\1 --sysroot=$EPREFIX;@" $installdir/compilers/bin/localrc + echo "set DEFLIBDIR=$EBROOTGENTOO/lib64;" >> $installdir/compilers/bin/localrc + echo "set DEFSTDOBJDIR=$EBROOTGENTOO/lib64;" >> $installdir/compilers/bin/localrc''' + old_opts = self.cfg['postinstallcmds'] + self.cfg.update('postinstallcmds', new_opts) + print_msg("Updated postinstall cmds from %s to %s", old_opts, new_opts) + else: + raise EasyBuildError("NVHPC-specific hook triggered for non-NVHPC easyconfig?!") + + def pre_test_hook(self, *args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" @@ -1218,6 +1238,7 @@ def post_module_hook(self, *args, **kwargs): 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, + 'NVHPC': pre_configure_hook_nvhpc_x86_64, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, 'LAMMPS': pre_configure_hook_LAMMPS_zen4, From 26b0965c7a08ac69fea562c834335deb6e9a4495 Mon Sep 17 00:00:00 2001 From: Adam McCartney Date: Tue, 29 Apr 2025 10:29:23 +0200 Subject: [PATCH 2/3] Remove EBROOTGENTOO variable Replaces EBROOTGENTOO with EPREFIX/usr --- eb_hooks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index d08a2b1608..31dea704df 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -753,10 +753,9 @@ def pre_configure_hook_nvhpc_x86_64(self, *args, **kwargs): eprefix = get_eessi_envvar("EPREFIX") new_opts = f'''installdir=%(installdir)s/Linux_x86_64/%(version)s EPREFIX={eprefix} - EBROOTGENTOO={eprefix}/usr sed -i "s@\(set LDSO=.*\);@\\1 --sysroot=$EPREFIX;@" $installdir/compilers/bin/localrc - echo "set DEFLIBDIR=$EBROOTGENTOO/lib64;" >> $installdir/compilers/bin/localrc - echo "set DEFSTDOBJDIR=$EBROOTGENTOO/lib64;" >> $installdir/compilers/bin/localrc''' + echo "set DEFLIBDIR=$EPREFIX/usr/lib64;" >> $installdir/compilers/bin/localrc + echo "set DEFSTDOBJDIR=$EPREFIX/usr/lib64;" >> $installdir/compilers/bin/localrc''' old_opts = self.cfg['postinstallcmds'] self.cfg.update('postinstallcmds', new_opts) print_msg("Updated postinstall cmds from %s to %s", old_opts, new_opts) From a2fe8bec8e70561e34e92273b696632fdb30f5d5 Mon Sep 17 00:00:00 2001 From: Adam McCartney Date: Mon, 5 May 2025 16:44:34 +0200 Subject: [PATCH 3/3] Ensure linker search paths are in correct order So I finally got this working (building to a custom cvmfs repo, then loading on a compute node and reproducing the sanity checks). There were a number of issues that needed to be worked out. 1. A potential issue that might appear if the linker happens to first find a script called "libc.so". The script is located in the compat layer and looks like it may possibly(?) redirect the linker to the host /lib64/libc.so.6 if it gets picked up. > cat $EPREFIX/usr/lib64/libc.so /* GNU ld script Use the shared library, but some functions are only in the static library, so try that secondarily. */ OUTPUT_FORMAT(elf64-x86-64) GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) ) 2. Another issue is that nvc++ will scan a number of directories looking for localrc files, if there are any old localrc files lying around that point to the wrong place, this will cause problems. The case below shows a situation where the localrc was pointing to a (removed) host_injections path >nvc++ -dryrun -std=c++20 minimal.cpp minimal Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/.nvc++rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/nativerc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/fnativerc Skipping rcfiles/internalrc (not found) Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/ccrc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/ccirc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/cpprc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/cppcurc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/paralgorc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/x86rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/x8664rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/lin86rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/lincommonrc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/lin8664rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/llvmcomprc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/llvmrc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/llvmx86rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/llvmx8664rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/omprc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/iparc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/acc1rc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/cudaselectrc Skipping rcfiles/persnvflangrc (not found) Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/acclin8664rc Skipping rcfiles/acctoolsrc (not found) Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/targetrc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/deprecatedrc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/c++llvmrc Skipping rcfiles/llvmxrc (not found) Skipping rcfiles/tunexrc (not found) Skipping rcfiles/clangxrc (not found) Skipping rcfiles/gccxrc (not found) Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/rcfiles/persnvirc Reading rcfile /cvmfs/software.asc.ac.at/versions/2023.06/software/linux/x86_64/amd/zen4/software/NVHPC/25.1-CUDA-12.6.0/Linux_x86_64/25.1/compilers/bin/localrc Skipping localrc.n3001-003 (not found) Reading rcfile /home/fs60000/admccartney/.config/NVIDIA/nvhpc/25.1/localrc.n3001-003 Skipping siterc (not found) Skipping siterc.n3001-003 (not found) Skipping $GCCLOCALRC (not found) Skipping .mynvrc (not found) Skipping .mynvc++rc (not found) Skipping .mynvcpprc (not found) Skipping .mynvx86rc (not found) Skipping $MYLOCALRC (not found) Skipping cudarc (not found) Action(realpath(/opt/acceptance-tests/eessi/2023.06/software/linux/x86_64/amd/zen4/software/GCCcore/13.3.0/bin/../lib/gcc/x86_64-pc-linux-gnu/13.3.0//../../../..)) Error in path /opt/acceptance-tests/eessi/2023.06/software/linux/x86_64/amd/zen4/software/GCCcore/13.3.0/bin/../lib/gcc/x86_64-pc-linux-gnu/13.3.0//../../../.. It should __not__ be finding the localrc files in my home directory {EESSI 2023.06} admccartney@n3001-003 ~/tests/nvhpc >rm -rf ~/.config/NVIDIA/nvhpc/25.1/ > nvc++ -std=c++20 minimal.cpp -o minimal >./minimal Hello world --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 31dea704df..6111ca2229 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -754,7 +754,7 @@ def pre_configure_hook_nvhpc_x86_64(self, *args, **kwargs): new_opts = f'''installdir=%(installdir)s/Linux_x86_64/%(version)s EPREFIX={eprefix} sed -i "s@\(set LDSO=.*\);@\\1 --sysroot=$EPREFIX;@" $installdir/compilers/bin/localrc - echo "set DEFLIBDIR=$EPREFIX/usr/lib64;" >> $installdir/compilers/bin/localrc + echo "set DEFLIBDIR=$EPREFIX/lib64:$EPREFIX/usr/lib64;" >> $installdir/compilers/bin/localrc echo "set DEFSTDOBJDIR=$EPREFIX/usr/lib64;" >> $installdir/compilers/bin/localrc''' old_opts = self.cfg['postinstallcmds'] self.cfg.update('postinstallcmds', new_opts)