nvhpc/25.3 + tiles

Author: lenzenbe

multi/Makefile_leonardo

@@ -14,7 +14,7 @@ EXTRA_DIR = /leonardo/prod/opt/compilers/cuda/12.3/none/compat
 export $(CUDECOMP_DIR)/lib
 
 # Compiler and Linker Flags
-FFLAGS = -fast -acc -gpu=managed -Minfo=accel -Mfree -Mpreprocess -cpp -cuda  -I$(CUDECOMP_DIR)/include/
+FFLAGS = -fast -acc -gpu=mem:managed -Minfo=accel -Mfree -Mpreprocess -cpp -cuda  -I$(CUDECOMP_DIR)/include/
 LIBS = -L$(CUDECOMP_DIR)/lib/ -L$(CUDA_DIR)/lib64/ -lcudecomp_fort -lcudecomp -cudalib=cufft  -lnvToolsExt
 
 # Source and Object Files

multi/go_leo_prof.sh

@@ -10,9 +10,12 @@
 #SBATCH --qos=boost_qos_dbg
 #SBATCH --error=test.err
 
-module load nvhpc/24.3
-module load cuda/12.3
-module load openmpi/4.1.6--nvhpc--24.3
+#module load nvhpc/24.3
+#module load cuda/12.3
+#module load openmpi/4.1.6--nvhpc--24.3
+module load profile/candidate
+module load nvhpc/25.3
+module load hpcx-mpi/2.19
 #export LD_LIBRARY_PATH=/leonardo_scratch/large/userexternal/aroccon0/MHIT36_cuDecomp/cuDecomp/build/lib:$LD_LIBRARY_PATH
 #export LD_LIBRARY_PATH=/leonardo_scratch/large/userexternal/lenzenbe/RE95_256_cuDec/cuDecomp/build/lib:$LD_LIBRARY_PATH
 CURRENT_DIR="$(pwd)"
@@ -25,7 +28,10 @@ export LD_LIBRARY_PATH=$ROOT_DIR:$LD_LIBRARY_PATH
 #mpirun -n 4 nsys profile --trace=cuda,nvtx,mpi -o profile_output_%q{SLURM_PROCID} --stats=true ./mhit36
 #mpirun  -n 4 nsys profile -t cuda,nvtx,mpi -o report.$SLURM_LOCALID ./mhit36
 #srun -n 4  nsys profile -t cuda,nvtx,mpi --output=nsys_report_rank%t ./mhit36nsys profile --multiprocess=true -t cuda,nvtx,mpi -o report $
-nsys profile  -t cuda,nvtx,mpi,openacc  mpirun -np 4 ./mhit36
+#nsys profile  -t cuda,nvtx,mpi,openacc  mpirun -np 4 ./mhit36
+
+# for nsight compute report
+mpirun -n 4 ncu --kernel-name main_659 --set=full --import-source=yes -o profile -f --launch-skip 3 --launch-count 1 "./mhit36"
 
 
 

multi/leo.sh

@@ -1,6 +1,10 @@
-module load nvhpc/24.3
-module load cuda/12.3
-module load openmpi/4.1.6--nvhpc--24.3
+# old modules - ! if you use these, changes in Makefile are needed from -gpu=mem:managed to -gpu=managed
+#module load nvhpc/24.3
+#module load cuda/12.3
+#module load openmpi/4.1.6--nvhpc--24.3
+module load profile/candidate
+module load nvhpc/25.3
+module load hpcx-mpi/2.19
 cp Makefile_leonardo Makefile
 make clean
 make

multi/main.f90

@@ -136,13 +136,13 @@ program main
 CHECK_CUDECOMP_EXIT(cudecompGridDescCreate(handle, grid_desc, config, options))
 
 ! Print information on configuration
-if (rank == 0) then
-   write(*,"(' Running on ', i0, ' x ', i0, ' process grid ...')") config%pdims(1), config%pdims(2)
-   write(*,"(' Using ', a, ' transpose backend ...')") &
-            cudecompTransposeCommBackendToString(config%transpose_comm_backend)
-   write(*,"(' Using ', a, ' halo backend ...')") &
-            cudecompHaloCommBackendToString(config%halo_comm_backend)
-endif
+!if (rank == 0) then
+!   write(*,"(' Running on ', i0, ' x ', i0, ' process grid ...')") config%pdims(1), config%pdims(2)
+!   write(*,"(' Using ', a, ' transpose backend ...')") &
+!            cudecompTransposeCommBackendToString(config%transpose_comm_backend)
+!   write(*,"(' Using ', a, ' halo backend ...')") &
+!            cudecompHaloCommBackendToString(config%halo_comm_backend)
+!endif
 
 
 ! Get pencil info for the grid descriptor in the physical space
@@ -584,7 +584,7 @@ program main
    ! Projection step, convective terms
    ! 5.1a Convective terms NS
    ! Loop on inner nodes
-   !$acc parallel loop collapse(3)
+   !$acc parallel loop tile(16,4,2)
    do k=1+halo_ext, piX%shape(3)-halo_ext
       do j=1+halo_ext, piX%shape(2)-halo_ext
          do i=1,nx
@@ -626,7 +626,7 @@ program main
    enddo
 
    ! 5.1b Compute viscous terms
-   !$acc parallel loop collapse(3)
+   !$acc parallel loop tile(16,4,2)
    do k=1+halo_ext, piX%shape(3)-halo_ext
       do j=1+halo_ext, piX%shape(2)-halo_ext
          do i=1,nx
@@ -656,7 +656,7 @@ program main
 
    ! 5.1c NS forcing
 
-   !$acc parallel loop collapse(3)
+   !$acc parallel loop tile(16,4,2)
    do k=1+halo_ext, piX%shape(3)-halo_ext
       do j=1+halo_ext, piX%shape(2)-halo_ext
          do i = 1, piX%shape(1)