Building and Running on ALCF Polaris

Tested with Omega_h master@2b0510dd and Kokkos develop@2d7715239.

environment

module use /soft/modulefiles
module load PrgEnv-gnu
module load cudatoolkit-standalone
module load spack-pe-base/0.7.1 cmake

Kokkos

bdir=build-kokkos-gcc12
cmake -S kokkos -B $bdir \
  -DBUILD_SHARED_LIBS=ON \
  -DCRAYPE_LINK_TYPE=dynamic \
  -DCMAKE_CXX_COMPILER=CC \
  -DKokkos_ARCH_AMPERE80=ON \
  -DKokkos_ENABLE_SERIAL=ON \
  -DKokkos_ENABLE_OPENMP=off \
  -DKokkos_ENABLE_CUDA=on \
  -DKokkos_ENABLE_CUDA_LAMBDA=on \
  -DKokkos_ENABLE_DEBUG=off \
  -DCMAKE_INSTALL_PREFIX=$bdir/install

cmake --build build-kokkos-gcc12/ -j 12 --target install

Omega_h

bdir=build-omegah-gcc12
cmake -S omega_h -B $bdir \
  -DCMAKE_INSTALL_PREFIX=$PWD/$bdir/install \
  -DBUILD_SHARED_LIBS=on \
  -DOmega_h_USE_Kokkos=on \
  -DKokkos_PREFIX=$PWD/build-kokkos-gcc12/install \
  -DOmega_h_CUDA_ARCH=80 \
  -DOmega_h_USE_MPI=on \
  -DBUILD_TESTING=on \
  -DCMAKE_CXX_COMPILER=CC

cmake --build build-omegah-gcc12/ -j 12

Test Omega_h

qsub -I -l select=1 -l filesystems=home:eagle -l walltime=1:00:00 -q debug -A <project>
# on the allocated node
# setup the environment described above
cd build-omegah-gcc12
ctest # all tests should pass