Merge branch 'MFlowCode:master' into rdma_mpi

Author: Malmahrouqi3

toolchain/bootstrap/modules.sh

@@ -24,8 +24,8 @@ if [ -v $u_c ]; then
     log   "$Y""Gatech$W:  Phoenix    (p)"
     log   "$R""Caltech$W: Richardson (r)"
     log   "$BR""Brown$W: Oscar (o)"
-    log   "$B""DoD$W:     Carpenter  (c) | Nautilus (n)"
-    log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): "
+    log   "$B""DoD$W:     Carpenter Cray (cc) | Carpenter GNU (c) |  Nautilus (n)"
+    log_n "($G""a$W/$G""f$W/$G""s$W/$G""w$W/$C""b$W/$C""e$CR/$C""d/$C""dai$CR/$Y""p$CR/$R""r$CR/$B""cc$CR/$B""c$CR/$B""n$CR/$BR""o"$CR"): "
     read u_c
     log
 fi
@@ -68,7 +68,7 @@ fi
 log "Loading modules (& env variables) for $M$COMPUTER$CR on $M$CG$CR"'s:'
 
 # Reset modules to default system configuration (unless Carpenter)
-if [ "$u_c" != 'c' ]; then
+if [ "$u_c" '!=' 'cc' ] && [ "$u_c" '!=' 'c' ]; then
     module reset > /dev/null 2>&1
     code="$?"
 
@@ -97,7 +97,7 @@ if [ $(echo "$VARIABLES" | grep = | wc -c) -gt 0 ]; then
 fi
 
 # Don't check for Cray paths on Carpenter, otherwise do check if they exist
-if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" != 'c' ]; then
+if [ ! -z ${CRAY_LD_LIBRARY_PATH+x} ] && [ "$u_c" '!=' 'c' ]; then
     ok "Found $M\$CRAY_LD_LIBRARY_PATH$CR. Prepending to $M\$LD_LIBRARY_PATH$CR."
     export LD_LIBRARY_PATH="$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH"
 fi

toolchain/modules

@@ -70,11 +70,15 @@ dai-all python cmake nvhpc-openmpi3/24.3 cuda
 dai-all CC=nvc CXX=nvc++ FC=nvfortran
 dai-gpu MFC_CUDA_CC=89,90
 
-c     DoD Carpenter
+c     DoD Carpenter (GNU)
 c-all python/3.12.1
 c-cpu compiler-rt/2024.2.0 ifort/2024.2.0 icc/2023.1.0 mpi/latest cmake/3.28.1-intel-2023.0.0
 c-cpu CC=gcc CXX=g++ FC=gfortran
 
+cc     DoD Carpenter (Cray)
+cc-all cray-python/3.11.7 craype-x86-rome 
+cc-cpu PrgEnv-cray/8.5.0 cray-fftw/3.3.10.8 cray-hdf5/1.14.3.1 cray-pals/1.4.0 cray-libpals/1.4.0
+
 n     DoD Nautilus
 n-all slurm
 n-cpu penguin/openmpi/4.1.5/gcc-8.5.0

toolchain/templates/carpenter-cray.mako

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+<%namespace name="helpers" file="helpers.mako"/>
+
+% if engine == 'batch':
+#PBS -l select=${nodes}:ncpus=192:mpiprocs=${tasks_per_node}
+#PBS -N "${name}"
+#PBS -l walltime=${walltime}
+% if partition:
+#PBS -q ${partition}
+% endif
+% if account:
+#PBS -A ${account}
+% endif
+% if email:
+#PBS -M ${email}
+#PBS -m abe
+% endif
+#PBS -o "${name}.out"
+#PBS -e "${name}.err"
+#PBS -V
+% endif
+
+${helpers.template_prologue()}
+
+ok ":) Loading modules:\n"
+cd "${MFC_ROOT_DIR}"
+. ./mfc.sh load -c cc -m ${'g' if gpu else 'c'}
+cd - > /dev/null
+echo
+
+% for target in targets:
+    ${helpers.run_prologue(target)}
+
+    % if not mpi:
+        (set -x; ${profiler} "${target.get_install_binpath(case)}")
+    % else:
+        (set -x; ${profiler}                              \
+            mpirun -np ${nodes*tasks_per_node}            \
+                   "${target.get_install_binpath(case)}")
+    % endif
+            
+    ${helpers.run_epilogue(target)}
+
+    echo
+% endfor
+
+${helpers.template_epilogue()}