sin,cos precalc

Author: lenzenbe

multi/go_leo_prof.sh

@@ -10,7 +10,7 @@
 #SBATCH --qos=boost_qos_dbg
 #SBATCH --error=test.err
 #### if mapping is on
-####SBATCH --cpus-per-task=8
+#SBATCH --cpus-per-task=8
 
 
 #module load nvhpc/24.3
@@ -38,10 +38,16 @@ export LD_LIBRARY_PATH=$ROOT_DIR:$LD_LIBRARY_PATH
 #srun -n 4  nsys profile -t cuda,nvtx,mpi --output=nsys_report_rank%t ./mhit36nsys profile --multiprocess=true -t cuda,nvtx,mpi -o report $
 
 # profile + node mapping + nic 
-#mpirun -np 4 --map-by node:PE=8 --rank-by core nsys profile  -t cuda,nvtx,mpi,openacc  --nic-metrics=true  ./binder.sh ./mhit36
+mpirun -np 4 --map-by node:PE=8 --rank-by core nsys profile  -t cuda,nvtx,mpi,openacc  --nic-metrics=true  ./binder.sh ./mhit36
 
 # for nsight compute report
 #mpirun -n 4 ncu --kernel-name main_659 --set=full --import-source=yes -o profile -f --launch-skip 3 --launch-count 1 "./mhit36"
 
+# for nsight compute report - all kernels
+# mpirun -n 4 ncu --kernel-name regex:main_ --set=full --import-source=yes --launch-skip 70 --launch-count 18 -o reportall.%p ./mhit36
+
+# for nsight compute report - all kernels + mapping + nic
+# mpirun -np 4 --map-by node:PE=8 --rank-by core ncu --kernel-name regex:main_ --set=full --import-source=yes --launch-skip 70 --launch-count 18 -o reportall.%p ./binder.sh ./mhit36
+
 
 

multi/main.f90

@@ -14,6 +14,8 @@ program main
 
 
 implicit none
+! timer for scaling test
+real :: t_start, t_end, elapsed
 ! grid dimensions
 integer :: comm_backend
 integer :: pr, pc
@@ -230,6 +232,14 @@ program main
 enddo
 ! allocate k_d on the device (later on remove and use OpenACC + managed memory?)
 allocate(kx_d, source=kx)
+
+allocate(mysin(nx), mycos(nx))
+do i=1,nx
+   ! compute here the sin to avoid multiple computations of sin
+   mysin(i)=sin(k0*x(i))
+   ! compute here the cos to avoid multiple computations of cos
+   mycos(i)=cos(k0*x(i))
+enddo
 !########################################################################################################################################
 ! 1. INITIALIZATION AND cuDECOMP AUTOTUNING : END
 !########################################################################################################################################
@@ -370,12 +380,12 @@ program main
 !Save initial fields (only if a fresh start)
 if (restart .eq. 0) then
    if (rank.eq.0) write(*,*) "Save initial fields"
-   call writefield(tstart,1)
-   call writefield(tstart,2)
-   call writefield(tstart,3)
+   !call writefield(tstart,1)
+   !call writefield(tstart,2)
+   !call writefield(tstart,3)
    !call writefield(tstart,4)
    #if phiflag == 1
-   call writefield(tstart,5)
+   !call writefield(tstart,5)
    #endif
 endif
 !########################################################################################################################################
@@ -399,6 +409,8 @@ program main
 gamma=1.d0*gumax
 !$acc data copyin(piX)
 !$acc data create(rhsu_o, rhsv_o, rhsw_o)
+!$acc data copyin(mysin, mycos)
+call cpu_time(t_start)
 ! Start temporal loop
 do t=tstart,tfin
     ! Create custom label for each marker
@@ -558,7 +570,7 @@ program main
    ! Projection step, convective terms
    ! 5.1a Convective terms NS
    ! Loop on inner nodes
-   !$acc parallel loop tile(16,4,2)
+   !$acc parallel loop tile(16,4,2) present(mysin, mycos)
    do k=1+halo_ext, piX%shape(3)-halo_ext
       do j=1+halo_ext, piX%shape(2)-halo_ext
          do i=1,nx
@@ -612,9 +624,12 @@ program main
             kg = piX%lo(3) + k - 1 
             jg = piX%lo(2) + j - 1 
             ! ABC forcing
-            rhsu(i,j,k)= rhsu(i,j,k) + f3*sin(k0*x(kg))+f2*cos(k0*x(jg))
-            rhsv(i,j,k)= rhsv(i,j,k) + f1*sin(k0*x(i))+f3*cos(k0*x(kg))
-            rhsw(i,j,k)= rhsw(i,j,k) + f2*sin(k0*x(jg))+f1*cos(k0*x(i))
+            ! rhsu(i,j,k)= rhsu(i,j,k) + f3*sin(k0*x(kg))+f2*cos(k0*x(jg))
+            rhsu(i,j,k)= rhsu(i,j,k) + f3*mysin(kg)+f2*mycos(jg)
+            ! rhsv(i,j,k)= rhsv(i,j,k) + f1*sin(k0*x(i))+f3*cos(k0*x(kg))
+            rhsv(i,j,k)= rhsv(i,j,k) + f1*mysin(i)+f3*mycos(kg)
+            ! rhsw(i,j,k)= rhsw(i,j,k) + f2*sin(k0*x(jg))+f1*cos(k0*x(i))
+            rhsw(i,j,k)= rhsw(i,j,k) + f2*mysin(jg)+f1*mycos(i)
             ! TG Forcing
             !rhsu(i,j,k)= rhsu(i,j,k) + f1*sin(k0*x(i))*cos(k0*x(j))*cos(k0*x(k))
             !rhsv(i,j,k)= rhsv(i,j,k) - f1*cos(k0*x(i))*sin(k0*x(j))*sin(k0*x(k))
@@ -1002,13 +1017,13 @@ program main
    if (mod(t,dump) .eq. 0) then
       if (rank .eq. 0) write(*,*) "Saving output files"
           ! write velocity and pressure fiels (1-4)
-         call writefield(t,1)
-         call writefield(t,2)
-         call writefield(t,3)
+         !call writefield(t,1)
+         !call writefield(t,2)
+         !call writefield(t,3)
          !call writefield(t,4)
          #if phiflag == 1
          ! write phase-field (5)
-         call writefield(t,5)
+         !call writefield(t,5)
          #endif
    endif
    !########################################################################################################################################
@@ -1018,12 +1033,16 @@ program main
 call nvtxEndRange
 !call nvtxEndRange
 enddo
+call cpu_time(t_end)
+elapsed = t_end-t_start
+if (rank .eq. 0) write(*,*)  'Elapsed time (seconds):', elapsed
+!$acc end data
 !$acc end data
 !$acc end data
 
 ! Remove allocated variables (add new)
 deallocate(u,v,w)
-deallocate(tanh_psi)
+deallocate(tanh_psi, mysin, mycos)
 deallocate(rhsu,rhsv,rhsw)
 deallocate(rhsu_o,rhsv_o,rhsw_o)
 deallocate(phi,rhsphi,normx,normy,normz)

multi/module.f90

@@ -51,6 +51,7 @@ module velocity
    double precision :: uc, vc, wc, umax, gumax=1.0d0, cou, alpha, beta
    double precision :: h11, h12, h13, h21, h22, h23, h31, h32, h33
    double precision :: umean, vmean, wmean, gumean, gvmean, gwmean
+   double precision, allocatable :: mysin(:), mycos(:)
 end module velocity