Mira and Cetus

Tau

Documentation

2015 Mira Performance Bootcamp presentation

Mira/Cetus/Vesta Tau User Guide

Setup

Run the following commands in each terminal/session.

soft add +tau-latest

Building

Create the toolchain file bgq-xl-tau.cmake with the following contents

set(CMAKE_SYSTEM_NAME BlueGeneQ-static)

set(COMPILER_SEARCH_PATHS 
/soft/perftools/tau/tau_latest/ppc64/bin/
/bgsys/drivers/ppcfloor/comm/xl/bin 
/soft/compilers/ibmcmp-aug2015/vac/bg/12.1/bin/
/soft/compilers/ibmcmp-aug2015/xlf/bg/14.1/bin/
/soft/compilers/ibmcmp-may2015/vacpp/bg/12.1/bin/)

find_program(CMAKE_C_COMPILER       tau_cc.sh    ${COMPILER_SEARCH_PATHS} )
find_program(CMAKE_CXX_COMPILER     tau_cxx.sh   ${COMPILER_SEARCH_PATHS} )
find_program(CMAKE_Fortran_COMPILER tau_f90.sh  ${COMPILER_SEARCH_PATHS} )

SET_PROPERTY(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS FALSE)
SET_PROPERTY(GLOBAL PROPERTY TARGET_ARCHIVES_MAY_BE_SHARED_LIBS FALSE)

set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)

#The combination of tau's use of the mpi compiler wrappers and our
# use of MPI_*_libraries from find_package(mpi) results in
# several 'multiple symbol definition' errors when linking phastaIC.exe.
#Override the variables set by find_package(mpi) to avoid find_package(mpi) 
# autodetection.  Setting the libraries to 'm' is a hack - FindMPI.cmake 
# does not want those vars to be empty strings.
set(MPI_Fortran_COMPILER mpixlf90)
set(MPI_Fortran_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64")
set(MPI_Fortran_LIBRARIES "m")
set(MPI_C_COMPILER mpixlc)
set(MPI_C_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64/comm/include")
set(MPI_C_LIBRARIES "m")
set(MPI_CXX_COMPILER mpixlcxx)
set(MPI_CXX_INCLUDE_PATH "/bgsys/drivers/V1R2M2/ppc64/comm/lib/xl")
set(MPI_CXX_LIBRARIES "m")

Create a cmake script config-xl-tau.sh with the following contents:

#!/bin/bash -ex
export TAU_MAKEFILE=$TAU/Makefile.tau-bgqtimers-papi-mpi-pdt
opt='-g -O2 -qstrict '
les=/path/to/libles.a
toolchain=/path/to/bgq-xl-tau.cmake

cmake .. \
--debug-output \
-DCMAKE_TOOLCHAIN_FILE=$toolchain \
-DCMAKE_C_FLAGS="$opt" \
-DCMAKE_CXX_FLAGS="$opt" \
-DCMAKE_Fortran_FLAGS="$opt" \
\
-DPHASTA_INCOMPRESSIBLE=ON \
-DPHASTA_COMPRESSIBLE=ON \
-DACUSOLVE_LIB=$les

Configure and make:

   chmod +x config-xl-tau.sh
   mkdir build-xl-tau
   cd build-xl-tau
   ./config-xl-tau.sh
   make #and wait

Running

Submit the job and run as you would normally. Optionally, Tau variables can be passed to qsub (can't get it working with runjob....) to control its behavior. For example, to merge profile data into a single file (strongly recommended) pass the following environment setting to qsub:

--env TAU_PROFILE_FORMAT="merged"

A sample execution script is below:

#!/bin/bash -ex
project=SC_Architectures
email=you@domain
processes=$((2**16))
ppn=64
nodes=$((processes/ppn))
startingtimestep=76800
#procscasedir=65536-procs_case-SyncIO-64-NoParma
procscasedir=65536-procs_case-SyncIO-64-Parma
exe=/home/cwsmith/projects/mrPhasta/build-tau/bin/phastaIC.exe
BG_OPTS="BG_SHAREDMEMSIZE=64MB:PAMID_VERBOSE=1"
TAU_OPTS="TAU_PROFILE_FORMAT=merged:PHASTA_CONFIG=/gpfs/mira-fs0/projects/UnsAdaptCFD_tesp/cwsmith/phastaParmaDec2015"
ln -snf $procscasedir $processes-procs_case
echo $startingtimestep > $processes-procs_case/numstart.dat
echo "processes=$processes ppn=$ppn nodes=$nodes startingtimestep=$startingtimestep "
qsub -t 30 -n $nodes --mode c${ppn} -A $project -M $email --env $TAU_OPTS --env $BG_OPTS $exe solver.mr.inp

Analysis

paraprof will report out of memory errors if it is run on a cetus/mira front end node on data from a 64K process case. Copy the tauprofile.xml file, ~260MB compressed for 64K processes, to a local system and run paraprof.