Fix Phoenix template for real (#824)

Co-authored-by: Benjamin Alexander Wilfong <bwilfong3@login-phoenix-rh9-6.pace.gatech.edu>

Author: wilfonba

src/simulation/m_time_steppers.fpp

@@ -1148,7 +1148,7 @@ contains
             if (proc_rank == 0) print *, 'RKCK 5th time-stage at', rkck_time_tmp
 #endif
             call s_compute_rhs(q_cons_ts(2)%vf, q_T_sf, q_prim_vf, rhs_ts_rkck(5)%vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
-            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(5)%vf, 5)
+            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(5)%vf, RKstep)
             call s_update_tmp_rkck(5, q_cons_ts, rhs_ts_rkck, lag_largestep)
             if (lag_largestep > 0._wp) call s_compute_rkck_dt(lag_largestep, restart_rkck_step)
             if (restart_rkck_step) cycle
@@ -1162,7 +1162,7 @@ contains
             if (proc_rank == 0) print *, 'RKCK 6th time-stage at', rkck_time_tmp
 #endif
             call s_compute_rhs(q_cons_ts(2)%vf, q_T_sf, q_prim_vf, rhs_ts_rkck(6)%vf, pb_ts(1)%sf, rhs_pb, mv_ts(1)%sf, rhs_mv, t_step, time_avg)
-            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(6)%vf, 6)
+            call s_compute_EL_coupled_solver(q_cons_ts(2)%vf, q_prim_vf, rhs_ts_rkck(6)%vf, RKstep)
             call s_update_tmp_rkck(6, q_cons_ts, rhs_ts_rkck, lag_largestep)
             if (lag_largestep > 0._wp) call s_compute_rkck_dt(lag_largestep, restart_rkck_step)
             if (restart_rkck_step) cycle

toolchain/templates/phoenix.mako

@@ -18,8 +18,7 @@
 #SBATCH --qos=${quality_of_service}
 % endif
 % if gpu:
-#SBATCH --gres=gpu:V100:${tasks_per_node}
-#SBATCH --mem-per-gpu=16G\
+#SBATCH -G${tasks_per_node}
 % endif
 % if email:
 #SBATCH --mail-user=${email}
@@ -41,10 +40,15 @@ echo
     % if not mpi:
         (set -x; ${profiler} "${target.get_install_binpath(case)}")
     % else:
-        (set -x; ${profiler}    \
-            mpirun -np ${nodes*tasks_per_node}            \
-                   --bind-to none                         \
-                   "${target.get_install_binpath(case)}")
+         (set -x; srun \
+            % if engine == 'interactive':
+                --nodes ${nodes} --ntasks-per-node ${tasks_per_node} \
+                --cpus-per-task 1                                    \
+                % if gpu:
+                    --gpus-per-task 1 --gpu-bind closest                 \
+                % endif
+            % endif
+            ${profiler} "${target.get_install_binpath(case)}")
     % endif
 
     ${helpers.run_epilogue(target)}