Build and Run on Aurora

The following script will build Kokkos and Omega_h on an aurora compute node. This was tested with scorec/omega_h @ 30cbd3a with the following Aurora environment:

Currently Loaded Modules:
  1) gcc-runtime/13.3.0-ghotoln (H)   4) mpc/1.3.1-rdrlvsl       (H)   7) libiconv/1.17-jjpb4sl         (H)  10) yaksa/0.3-7ks5f26             (H)  13) cray-pals/1.4.0     16) cmake/3.30.5
  2) gmp/6.3.0-mtokfaw          (H)   5) gcc/13.3.0                    8) libxml2/2.13.5                     11) mpich/opt/develop-git.6037a7a      14) cray-libpals/1.4.0
  3) mpfr/4.2.1-gkcdl5w         (H)   6) oneapi/release/2025.0.5       9) hwloc/2.11.3-mpich-level-zero      12) libfabric/1.22.0                   15) gmake/4.4.1

#!/bin/bash
 
root=`date "+%m-%d-%Y_%H.%M.%S"`
mkdir $root
cd $root
 
rm -rf omegahKkSycl
mkdir omegahKkSycl
cd omegahKkSycl
git clone https://github.com/SCOREC/omega_h.git
git clone -b develop https://github.com/Kokkos/kokkos.git

module load cmake

#use explicit scaling
export EnableImplicitScaling=0
export ZE_AFFINITY_MASK=0.0
 
bdir=$PWD/buildKokkosSycl
 
cmake -S kokkos -B $bdir \
 -DCMAKE_BUILD_TYPE=Release\
 -DCMAKE_CXX_COMPILER=icpx\
 -DCMAKE_CXX_EXTENSIONS=OFF\
 -DCMAKE_CXX_STANDARD=17\
 -DKokkos_ENABLE_TESTS=OFF\
 -DKokkos_ENABLE_SERIAL=ON\
 -DKokkos_ENABLE_OPENMP=OFF\
 -DKokkos_ENABLE_SYCL=ON\
 -DKokkos_ARCH_INTEL_PVC=ON\
 -DBUILD_SHARED_LIBS=OFF\
 -DKokkos_ENABLE_DEBUG=OFF\
 -DKokkos_ENABLE_EXAMPLES=OFF\
 -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations -Wno-tautological-constant-compare"\
 -DCMAKE_EXE_LINKER_FLAGS="-fsycl-max-parallel-link-jobs=48 -Xsycl-target-backend '-device pvc -options -ze-intel-enable-auto-large-GRF-mode'" \
 -DCMAKE_VERBOSE_MAKEFILE=OFF\
 -DCMAKE_INSTALL_PREFIX=$bdir/install
 
cmake --build $bdir --target install -j32
 
bdir=$PWD/buildOmegahKkSyclAot
 
cmake -S omega_h -B $bdir \
-DCMAKE_INSTALL_PREFIX=$bdir/install \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_CXX_COMPILER=mpicxx \
-DCMAKE_EXE_LINKER_FLAGS="-fsycl-max-parallel-link-jobs=48 -Xsycl-target-backend '-device pvc -options -ze-intel-enable-auto-large-GRF-mode'" \
-DBUILD_SHARED_LIBS=OFF \
-DOmega_h_USE_CUDA=OFF \
-DOmega_h_USE_OpenMP=OFF \
-DOmega_h_USE_MPI=ON \
-DOmega_h_USE_Kokkos=ON \
-DKokkos_PREFIX=$PWD/buildKokkosSycl/install \
-DBUILD_TESTING=ON
 
#build all the tests, takes ~30mins on aurora
cmake --build $bdir -j32

#run the tests
ctest --test-dir $bdir --no-tests=error --timeout 40 --repeat until-fail:5

Delta Wing Benchmark

Download the Omega_h delta wing meshes: https://zenodo.org/records/10672130

Run script for a single tile:

#!/bin/bash
bin=/path/to/omegahKkSycl/buildOmegahKkSyclAot/src
mesh=/path/to/deltaWing_500kMetric.osh
for implicitScaling in On Off; do 
  [ "$implicitScaling" == Off ] && export EnableImplicitScaling=0 && export ZE_AFFINITY_MASK=0.0
  [ "$implicitScaling" == On ] && unset EnableImplicitScaling && unset ZE_AFFINITY_MASK
  for r in {0..3}; do 
      case=500k
      for opt in noTimePool time pool timePool; do
        arg=""
        [ "$opt" == "time" ] && arg="--osh-time" && export ZE_SERIALIZE=2
        [ "$opt" == "pool" ] && arg="--osh-pool" && unset ZE_SERIALIZE
        [ "$opt" == "timePool" ] && arg="--osh-time --osh-pool" && export ZE_SERIALIZE=2
        set -x
        logFile=${case}-${opt}-run${r}-implicitScaling${implicitScaling}.log
        echo $logFile
        $bin/ugawg_hsc_oshmeshload $arg $mesh &> ${logFile}
        set +x
      done #opt
  done #runs
done #scaling