From 4ee44512b826ce854a5ba9365f77b818e09cc990 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Fri, 9 Aug 2024 16:22:55 -0600 Subject: [PATCH 1/5] Update README.md with zenodo link --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 61d9e76..2b8fd74 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # FastEddy® ©2016 University Corporation for Atmospheric Research +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.11042754.svg)](https://doi.org/10.5281/zenodo.11042754) + # Open-source License The FastEddy® model is licensed under the Apache License, Version 2.0 (the "License"); you may not use any source code in this repository except in compliance with the License. From cf225adb6b8c0ab661aff6f4fb6ea9299297516f Mon Sep 17 00:00:00 2001 From: Joe Schoonover Date: Sat, 18 Jan 2025 12:25:57 -0500 Subject: [PATCH 2/5] Add makefile for building for AMD GPUs with hip --- SRC/FEMAIN/Makefile.hip | 213 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 SRC/FEMAIN/Makefile.hip diff --git a/SRC/FEMAIN/Makefile.hip b/SRC/FEMAIN/Makefile.hip new file mode 100644 index 0000000..cd7cc6a --- /dev/null +++ b/SRC/FEMAIN/Makefile.hip @@ -0,0 +1,213 @@ +################################################################################ +# FastEddy®: SRC/FEMAIN/Makefile +# ©2016 University Corporation for Atmospheric Research +# +# This file is licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ################################################################################ +# +# ################################################################################ +# # Available build targets +# ################################################################################ + +TARGETS = \ + FastEddy + + +################################################################################ +## Compiler and flags +################################################################################ + +TEST_CC = mpicc +TEST_CU_CC = hipcc -Rpass-analysis=kernel-resource-usage --gpu-max-threads-per-block=256 +ROCM_PATH ?= /opt/rocm +MPI_ROOT ?= "" +NETCDF_C_ROOT ?= "" +GPU_ARCH ?= gfx90a + +# For compilation on NCAR's Casper or Derecho supercomputers use the OTHER_INCLUDES below +OTHER_INCLUDES = -I${MPI_ROOT}/include + +DEBUG_CFLAGS = -g + +DEFINES = -DCUB_IGNORE_DEPRECATED_CPP_DIALECT -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT + +TEST_CFLAGS = -Wall -m64 $(shell ${ROCM_PATH}/bin/hipconfig --cpp_config) ${DEFINES} ${INCLUDES} ${OTHER_INCLUDES} +ARCH_CU_FLAGS = --offload-arch=${GPU_ARCH} +TEST_CU_CFLAGS = ${ARCH_CU_FLAGS} -m64 -std=c++14 ${DEFINES} ${INCLUDES} ${OTHER_INCLUDES} + +L_CPPFLAGS = + + +TEST_LDFLAGS = -L. +TEST_CU_LDFLAGS = -L. -fgpu-rdc + +TEST_LIBS = -lm -L${MPI_ROOT}/lib -lmpi -lstdc++ -L${ROCM_PATH}/lib -lamdhip64 -lhiprand -L${NETCDF_C_ROOT}/lib -lnetcdf +TEST_CU_LIBS = -lmpi -L${ROCM_PATH}/lib -lhiprtc + + +################################################################################ +# Sub-directories to INCLUDE +# ################################################################################ + +INCLUDES = \ + -I. \ + -I../TIME_INTEGRATION/ \ + -I../TIME_INTEGRATION/CUDA \ + -I../HYDRO_CORE \ + -I../HYDRO_CORE/CUDA \ + -I../GRID \ + -I../GRID/CUDA \ + -I../FECUDA \ + -I../IO/ \ + -I../MEM_UTILS/ \ + -I../FEMPI/ \ + -I../PARAMETERS/ + +################################################################################ +# Headers +# ############################################################################### + +TEST_HDRS = \ + ../TIME_INTEGRATION/time_integration.h \ + ../TIME_INTEGRATION/CUDA/cuda_timeInt.h \ + ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice_cu.h \ + ../HYDRO_CORE/hydro_core.h \ + ../HYDRO_CORE/CUDA/cuda_hydroCore.h \ + ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice_cu.h \ + ../GRID/grid.h \ + ../GRID/CUDA/cuda_grid.h \ + ../GRID/CUDA/cuda_gridDevice_cu.h \ + ../FECUDA/fecuda.h \ + ../FECUDA/fecuda_Device_cu.h \ + ../IO/io.h \ + ../MEM_UTILS/mem_utils.h \ + ../FEMPI/fempi.h \ + ../PARAMETERS/parameters.h \ + ../PARAMETERS/hashTable.h + +################################################################################ +# Objects +# ############################################################################### + +TEST_LIB_OBJS = + +TEST_OBJS = ./FastEddy.o \ + ../TIME_INTEGRATION/time_integration.o \ + ../TIME_INTEGRATION/CUDA/cuda_timeInt.o \ + ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.o \ + ../HYDRO_CORE/hydro_core.o \ + ../HYDRO_CORE/CUDA/cuda_hydroCore.o \ + ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.o \ + ../GRID/grid.o \ + ../GRID/CUDA/cuda_grid.o \ + ../GRID/CUDA/cuda_gridDevice.o \ + ../FECUDA/fecuda.o \ + ../FECUDA/fecuda_Device.o \ + ../IO/io.o \ + ../IO/ioVarsList.o \ + ../MEM_UTILS/mem_utils.o \ + ../FEMPI/fempi.o \ + ../PARAMETERS/parameters.o \ + ../PARAMETERS/hashTable.o + +################################################################################ +# Targets +# ############################################################################### + +all: hipify FastEddy + +hipify: + find ../ -name "*.cu" -exec hipify-perl -inplace {} \; + find ../ -name "*.h" -exec hipify-perl -inplace {} \; + find ../ -name "*.prehip" -exec rm {} \; + + +################################################################################ +# Generic compile rules +# ################################################################################ +%.o: %.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ + +.c.o: + ${COMP_FLAGS} -c $< -o $@ + +################################################################################ +# Module=specific compile rules +# ################################################################################ + +../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.o: ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.cu \ + ../TIME_INTEGRATION/CUDA/cuda_RKschemes.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ + +../IO/io.o: ../IO/io.c \ + ../IO/io_netcdf.c \ + ../IO/io_binary.c + $(COMP_FLAGS) -c $< -o $@ + +../FECUDA/fecuda_Device.o: ../FECUDA/fecuda_Device.cu \ + ../FECUDA/fecuda_Utils.cu \ + ../FECUDA/fecuda_PlugIns.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ + +../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.o: ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_BaseStateDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_buoyancyDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_coriolisDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_pressureDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_BCsDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_rayleighDampingDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_advectionDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_molecularDiffDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_surfaceLayerDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_sgsTurbDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_sgstkeDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_canopyDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_largeScaleForcingsDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_moistureDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_filtersDevice.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ +################################################################################ +# Generic Executable +# ############################################################################### + +FastEddy: COMP_FLAGS = ${TEST_CC} ${TEST_CFLAGS} + +TEST_DEPENDENCIES = ${TEST_OBJS} ${TEST_HDRS} FastEddy_devlink.o + +FastEddy_devlink.o: + $(TEST_CU_CC) ${ARCH_CU_FLAGS} ${TEST_OBJS} ${TEST_LDFLAGS} ${TEST_LIBS} -fgpu-rdc --hip-link --emit-static-lib -o FastEddy_devlink.o +FastEddy: ${TEST_DEPENDENCIES} + ${COMP_FLAGS} -o FastEddy ${TEST_OBJS} ./FastEddy_devlink.o ${TEST_LDFLAGS} ${TEST_LIBS} + +################################################################################ +# Clean +# ################################################################################ + +clean: + rm -rf ${TARGETS} ${TEST_OBJS} ${TEST_LIB_OBJS} ./FastEddy_devlink.o + +################################################################################ +# Check +# ############################################################################### + +check: + @echo CC = ${CC} + @echo CFLAGS = ${CFLAGS} + @echo LDFLAGS = ${LDFLAGS} + @echo LIBS = ${LIBS} + @echo DEPENDENCIES = ${TEST_DEPENDENCIES} From da39ab03304e899628911293984b6b73d4f088a8 Mon Sep 17 00:00:00 2001 From: Joe Schoonover Date: Sat, 18 Jan 2025 12:41:54 -0500 Subject: [PATCH 3/5] Add simple build docs for AMD GPUs --- docs/index.rst | 1 + docs/run_amd_gpus.rst | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100644 docs/run_amd_gpus.rst diff --git a/docs/index.rst b/docs/index.rst index 78ca016..a2e7199 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ FastEddy should be cited as shown here: release_notes.rst run_ncar_hpcs.rst + run_amd_gpus.rst downloads.rst Tutorials/index publications.rst diff --git a/docs/run_amd_gpus.rst b/docs/run_amd_gpus.rst new file mode 100644 index 0000000..252a20f --- /dev/null +++ b/docs/run_amd_gpus.rst @@ -0,0 +1,37 @@ +.. _run_fasteddy_amdgpu: + +************************** +Building FastEddy on AMD GPU Accelerated platforms +************************** + +These instructions will help users get started with building FastEddy for systems with AMD GPU accelerators. This can be beneficial for users who have allocations on + + * `Oak Ridge National Laboratory's Frontier `_ + * `Pawsey Supercomputing Centre's Setonix `_ + * `CSCS's Lumi _` + * `Fluid Numerics' Galapagos `_ + + +Compilation +=========== + +FastEddy requires a C-compiler, MPI, and CUDA. + + 1. Download the source code from the `Releases `_ page and unpack the release in the desired location or clone the `repository `_ in the desired location. + + 2. Navigate to the **SRC/FEMAIN** directory. + + 3. To build the FastEddy executable run :code:`make -f Makfile.hip` (optionally run :code:`make clean` first if appropriate). + +You may need to define a few environment variables that influence the build process to properly set the paths to various dependencies and to select the target GPU. + + * :code:`ROCM_PATH` : This is the path to your ROCm installation. This variable defaults to :code:`/opt/rocm`. However, on some systems, multiple versions of ROCm may be available via environment modules and this variable may need to be adjusted accordingly. + * :code:`MPI_ROOT` : This is the path to your MPI installation. Since this environment variable is not necessarily defined through an HPC center's environment modules, it is recommended that you set this variable appropriately. + * :code:`NETCDF_C_ROOT`: This is the path to your NetCDF-C installation. Since this environment variable is not necessarily defined through an HPC center's environment modules, it is recommended that you set this variable appropriately. + * :code:`GPU_ARCH` : This is the AMD GPU architecture code for the target GPU you want to build for. This variable defaults to :code:`gfx90a`, which corresponds to the MI210, MI250, and MI250X GPUs. + +The :code:`FastEddy` executable will be located in the **SRC/FEMAIN** directory. To +build on other HPC systems with NVIDIA GPUs, check for availability of the aformentioned +modules/dependencies. Successful compilation may require modifications to shell environment +variable include or library paths, or alternatively minor adjustments to the include or library +flags in **SRC/FEMAIN/Makefile.hip**. From d3ac68a4ee1ed2a092d633061b03060fb13c034c Mon Sep 17 00:00:00 2001 From: Joe Schoonover <11430768+fluidnumerics-joe@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:29:49 -0700 Subject: [PATCH 4/5] Update docs/run_amd_gpus.rst Co-authored-by: Julie Prestopnik --- docs/run_amd_gpus.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/run_amd_gpus.rst b/docs/run_amd_gpus.rst index 252a20f..a1891aa 100644 --- a/docs/run_amd_gpus.rst +++ b/docs/run_amd_gpus.rst @@ -1,6 +1,6 @@ .. _run_fasteddy_amdgpu: -************************** +************************************************** Building FastEddy on AMD GPU Accelerated platforms ************************** From 338190ba0548ef04e2548216a6713cb40bbd1d91 Mon Sep 17 00:00:00 2001 From: Joe Schoonover <11430768+fluidnumerics-joe@users.noreply.github.com> Date: Mon, 27 Jan 2025 09:29:57 -0700 Subject: [PATCH 5/5] Update docs/run_amd_gpus.rst Co-authored-by: Julie Prestopnik --- docs/run_amd_gpus.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/run_amd_gpus.rst b/docs/run_amd_gpus.rst index a1891aa..434293a 100644 --- a/docs/run_amd_gpus.rst +++ b/docs/run_amd_gpus.rst @@ -2,7 +2,7 @@ ************************************************** Building FastEddy on AMD GPU Accelerated platforms -************************** +************************************************** These instructions will help users get started with building FastEddy for systems with AMD GPU accelerators. This can be beneficial for users who have allocations on