Merge pull request #31 from JuliaParallel/initial

Add Preferences package in any case

Author: omlins

examples/cscs/daint/gpu/craype_config_no_cudaaware

@@ -10,19 +10,36 @@
 # Load required modules, including correct CPU and GPU target modules (NOTE: the same modules should be loaded when running julia - JUHPC can be used in a module build recipe...)
 module load daint-gpu
 module switch PrgEnv-cray PrgEnv-gnu
+module load cudatoolkit # A toolkit needs to be loaded during installation in order to be able to set JUHPC_CUDA_HOME (even though we set preferences to use artifacts in the postinstall)
 module load cray-hdf5-parallel
 module list
 
 
 # Environment variables for HPC key packages that require system libraries that require system libraries (MPI.jl, CUDA.jl, AMDGPU.jl, HDF5.jl and ADIOS2.jl)
+export JUHPC_CUDA_HOME=$CUDA_HOME
+export JUHPC_CUDA_RUNTIME_VERSION="11.8" # Set to the highest possible version that can work on daint (using artifacts - below "local" is set to false...)
 export JUHPC_MPI_HOME=$MPICH_DIR
 export JUHPC_MPI_EXEC="srun -C gpu"
 export JUHPC_HDF5_HOME=$HDF5_DIR
 
 
+# Create site-specific post-install script (currently MPIPreferences does not provide an option to set required preloads if not automatically detected; JUHPC_MPI_VENDOR fails on Piz Daint...)
+JUHPC_POST_INSTALL_JL=./post_install.jl
+echo 'using Preferences
+    set_preferences!("CUDA_Runtime_jll",
+        "local" => false;
+        force=true
+    )
+    set_preferences!("MPIPreferences",
+        "preloads" => ["libcuda.so", "libcudart.so"],
+        "preloads_env_switch" => "MPICH_RDMA_ENABLED_CUDA";
+        force=true
+    )' > $JUHPC_POST_INSTALL_JL
+
+
 # Call JUHPC
 JUHPC_SETUP_INSTALLDIR=$SCRATCH/../julia/${HOSTNAME%%[0-9]*}-gpu-nocudaaware/juhpc_setup
 JULIAUP_INSTALLDIR="\$SCRATCH/../julia/\$USER/\${HOSTNAME%%[0-9]*}-gpu-nocudaaware/juliaup"
 VERSION="v0.2.0"
 wget https://raw.githubusercontent.com/JuliaParallel/JUHPC/$VERSION/juhpc -O /tmp/juhpc
-bash -l /tmp/juhpc $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR
+bash -l /tmp/juhpc $JUHPC_SETUP_INSTALLDIR $JULIAUP_INSTALLDIR $JUHPC_POST_INSTALL_JL

juhpc

@@ -194,10 +194,8 @@ mkdir -p "$JULIA_PREFDIR" || { error "failed to create directory: $JULIA_PREFDIR
 progress_bar 1 # Initialize progress bar.
 julia_pref 'using Pkg' 10 # Initialize project.
 
-if [[ -n "${JUHPC_CUDA_HOME}" || -n "${JUHPC_ROCM_HOME}" ]]; then
-    julia_pref 'using Pkg; Pkg.add("Preferences")' 30
-    echo "[extras]" >> "$JULIA_PREF_PROJECT"
-fi
+julia_pref 'using Pkg; Pkg.add("Preferences")' 30 # Add Preferences package in any case to avoid that it has to be installed in postinstall scripts that only want to set preferences.
+echo "[extras]" >> "$JULIA_PREF_PROJECT"
 
 if [ -n "${JUHPC_CUDA_HOME}" ]; then  # Set preference for using the local CUDA runtime before any installation of CUDA.jl to avoid downloading of artifacts
     echo 'CUDA_Runtime_jll = "76a88914-d11a-5bdc-97e0-2f5a05c973a2"' >> "$JULIA_PREF_PROJECT"