From 2e91db3138282bcabfd4d2d53aeb18fb5a61b375 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Fri, 9 Aug 2024 16:23:15 -0600 Subject: [PATCH 1/7] Update README.md with Zenodo link --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 61d9e76..2b8fd74 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # FastEddy® ©2016 University Corporation for Atmospheric Research +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.11042754.svg)](https://doi.org/10.5281/zenodo.11042754) + # Open-source License The FastEddy® model is licensed under the Apache License, Version 2.0 (the "License"); you may not use any source code in this repository except in compliance with the License. From 0ab006f5937f0b3c056c5c659dc0cc581f253301 Mon Sep 17 00:00:00 2001 From: Joe Schoonover <11430768+fluidnumerics-joe@users.noreply.github.com> Date: Mon, 27 Jan 2025 15:42:43 -0500 Subject: [PATCH 2/7] Amdgpu support (#63) * Update README.md with zenodo link * Add makefile for building for AMD GPUs with hip * Add simple build docs for AMD GPUs * Update docs/run_amd_gpus.rst Co-authored-by: Julie Prestopnik * Update docs/run_amd_gpus.rst Co-authored-by: Julie Prestopnik --------- Co-authored-by: Julie Prestopnik --- SRC/FEMAIN/Makefile.hip | 213 ++++++++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/run_amd_gpus.rst | 37 +++++++ 3 files changed, 251 insertions(+) create mode 100644 SRC/FEMAIN/Makefile.hip create mode 100644 docs/run_amd_gpus.rst diff --git a/SRC/FEMAIN/Makefile.hip b/SRC/FEMAIN/Makefile.hip new file mode 100644 index 0000000..cd7cc6a --- /dev/null +++ b/SRC/FEMAIN/Makefile.hip @@ -0,0 +1,213 @@ +################################################################################ +# FastEddy®: SRC/FEMAIN/Makefile +# ©2016 University Corporation for Atmospheric Research +# +# This file is licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ################################################################################ +# +# ################################################################################ +# # Available build targets +# ################################################################################ + +TARGETS = \ + FastEddy + + +################################################################################ +## Compiler and flags +################################################################################ + +TEST_CC = mpicc +TEST_CU_CC = hipcc -Rpass-analysis=kernel-resource-usage --gpu-max-threads-per-block=256 +ROCM_PATH ?= /opt/rocm +MPI_ROOT ?= "" +NETCDF_C_ROOT ?= "" +GPU_ARCH ?= gfx90a + +# For compilation on NCAR's Casper or Derecho supercomputers use the OTHER_INCLUDES below +OTHER_INCLUDES = -I${MPI_ROOT}/include + +DEBUG_CFLAGS = -g + +DEFINES = -DCUB_IGNORE_DEPRECATED_CPP_DIALECT -DTHRUST_IGNORE_DEPRECATED_CPP_DIALECT + +TEST_CFLAGS = -Wall -m64 $(shell ${ROCM_PATH}/bin/hipconfig --cpp_config) ${DEFINES} ${INCLUDES} ${OTHER_INCLUDES} +ARCH_CU_FLAGS = --offload-arch=${GPU_ARCH} +TEST_CU_CFLAGS = ${ARCH_CU_FLAGS} -m64 -std=c++14 ${DEFINES} ${INCLUDES} ${OTHER_INCLUDES} + +L_CPPFLAGS = + + +TEST_LDFLAGS = -L. +TEST_CU_LDFLAGS = -L. -fgpu-rdc + +TEST_LIBS = -lm -L${MPI_ROOT}/lib -lmpi -lstdc++ -L${ROCM_PATH}/lib -lamdhip64 -lhiprand -L${NETCDF_C_ROOT}/lib -lnetcdf +TEST_CU_LIBS = -lmpi -L${ROCM_PATH}/lib -lhiprtc + + +################################################################################ +# Sub-directories to INCLUDE +# ################################################################################ + +INCLUDES = \ + -I. \ + -I../TIME_INTEGRATION/ \ + -I../TIME_INTEGRATION/CUDA \ + -I../HYDRO_CORE \ + -I../HYDRO_CORE/CUDA \ + -I../GRID \ + -I../GRID/CUDA \ + -I../FECUDA \ + -I../IO/ \ + -I../MEM_UTILS/ \ + -I../FEMPI/ \ + -I../PARAMETERS/ + +################################################################################ +# Headers +# ############################################################################### + +TEST_HDRS = \ + ../TIME_INTEGRATION/time_integration.h \ + ../TIME_INTEGRATION/CUDA/cuda_timeInt.h \ + ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice_cu.h \ + ../HYDRO_CORE/hydro_core.h \ + ../HYDRO_CORE/CUDA/cuda_hydroCore.h \ + ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice_cu.h \ + ../GRID/grid.h \ + ../GRID/CUDA/cuda_grid.h \ + ../GRID/CUDA/cuda_gridDevice_cu.h \ + ../FECUDA/fecuda.h \ + ../FECUDA/fecuda_Device_cu.h \ + ../IO/io.h \ + ../MEM_UTILS/mem_utils.h \ + ../FEMPI/fempi.h \ + ../PARAMETERS/parameters.h \ + ../PARAMETERS/hashTable.h + +################################################################################ +# Objects +# ############################################################################### + +TEST_LIB_OBJS = + +TEST_OBJS = ./FastEddy.o \ + ../TIME_INTEGRATION/time_integration.o \ + ../TIME_INTEGRATION/CUDA/cuda_timeInt.o \ + ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.o \ + ../HYDRO_CORE/hydro_core.o \ + ../HYDRO_CORE/CUDA/cuda_hydroCore.o \ + ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.o \ + ../GRID/grid.o \ + ../GRID/CUDA/cuda_grid.o \ + ../GRID/CUDA/cuda_gridDevice.o \ + ../FECUDA/fecuda.o \ + ../FECUDA/fecuda_Device.o \ + ../IO/io.o \ + ../IO/ioVarsList.o \ + ../MEM_UTILS/mem_utils.o \ + ../FEMPI/fempi.o \ + ../PARAMETERS/parameters.o \ + ../PARAMETERS/hashTable.o + +################################################################################ +# Targets +# ############################################################################### + +all: hipify FastEddy + +hipify: + find ../ -name "*.cu" -exec hipify-perl -inplace {} \; + find ../ -name "*.h" -exec hipify-perl -inplace {} \; + find ../ -name "*.prehip" -exec rm {} \; + + +################################################################################ +# Generic compile rules +# ################################################################################ +%.o: %.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ + +.c.o: + ${COMP_FLAGS} -c $< -o $@ + +################################################################################ +# Module=specific compile rules +# ################################################################################ + +../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.o: ../TIME_INTEGRATION/CUDA/cuda_timeIntDevice.cu \ + ../TIME_INTEGRATION/CUDA/cuda_RKschemes.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ + +../IO/io.o: ../IO/io.c \ + ../IO/io_netcdf.c \ + ../IO/io_binary.c + $(COMP_FLAGS) -c $< -o $@ + +../FECUDA/fecuda_Device.o: ../FECUDA/fecuda_Device.cu \ + ../FECUDA/fecuda_Utils.cu \ + ../FECUDA/fecuda_PlugIns.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ + +../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.o: ../HYDRO_CORE/CUDA/cuda_hydroCoreDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_BaseStateDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_buoyancyDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_coriolisDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_pressureDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_BCsDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_rayleighDampingDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_advectionDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_molecularDiffDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_surfaceLayerDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_sgsTurbDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_sgstkeDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_canopyDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_largeScaleForcingsDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_moistureDevice.cu \ + ../HYDRO_CORE/CUDA/cuda_filtersDevice.cu + rm -rf ./FastEddy_devlink.o; \ + $(TEST_CU_CC) $(TEST_CU_CFLAGS) -fgpu-rdc -c $< -o $@ +################################################################################ +# Generic Executable +# ############################################################################### + +FastEddy: COMP_FLAGS = ${TEST_CC} ${TEST_CFLAGS} + +TEST_DEPENDENCIES = ${TEST_OBJS} ${TEST_HDRS} FastEddy_devlink.o + +FastEddy_devlink.o: + $(TEST_CU_CC) ${ARCH_CU_FLAGS} ${TEST_OBJS} ${TEST_LDFLAGS} ${TEST_LIBS} -fgpu-rdc --hip-link --emit-static-lib -o FastEddy_devlink.o +FastEddy: ${TEST_DEPENDENCIES} + ${COMP_FLAGS} -o FastEddy ${TEST_OBJS} ./FastEddy_devlink.o ${TEST_LDFLAGS} ${TEST_LIBS} + +################################################################################ +# Clean +# ################################################################################ + +clean: + rm -rf ${TARGETS} ${TEST_OBJS} ${TEST_LIB_OBJS} ./FastEddy_devlink.o + +################################################################################ +# Check +# ############################################################################### + +check: + @echo CC = ${CC} + @echo CFLAGS = ${CFLAGS} + @echo LDFLAGS = ${LDFLAGS} + @echo LIBS = ${LIBS} + @echo DEPENDENCIES = ${TEST_DEPENDENCIES} diff --git a/docs/index.rst b/docs/index.rst index 78ca016..a2e7199 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -30,6 +30,7 @@ FastEddy should be cited as shown here: release_notes.rst run_ncar_hpcs.rst + run_amd_gpus.rst downloads.rst Tutorials/index publications.rst diff --git a/docs/run_amd_gpus.rst b/docs/run_amd_gpus.rst new file mode 100644 index 0000000..434293a --- /dev/null +++ b/docs/run_amd_gpus.rst @@ -0,0 +1,37 @@ +.. _run_fasteddy_amdgpu: + +************************************************** +Building FastEddy on AMD GPU Accelerated platforms +************************************************** + +These instructions will help users get started with building FastEddy for systems with AMD GPU accelerators. This can be beneficial for users who have allocations on + + * `Oak Ridge National Laboratory's Frontier `_ + * `Pawsey Supercomputing Centre's Setonix `_ + * `CSCS's Lumi _` + * `Fluid Numerics' Galapagos `_ + + +Compilation +=========== + +FastEddy requires a C-compiler, MPI, and CUDA. + + 1. Download the source code from the `Releases `_ page and unpack the release in the desired location or clone the `repository `_ in the desired location. + + 2. Navigate to the **SRC/FEMAIN** directory. + + 3. To build the FastEddy executable run :code:`make -f Makfile.hip` (optionally run :code:`make clean` first if appropriate). + +You may need to define a few environment variables that influence the build process to properly set the paths to various dependencies and to select the target GPU. + + * :code:`ROCM_PATH` : This is the path to your ROCm installation. This variable defaults to :code:`/opt/rocm`. However, on some systems, multiple versions of ROCm may be available via environment modules and this variable may need to be adjusted accordingly. + * :code:`MPI_ROOT` : This is the path to your MPI installation. Since this environment variable is not necessarily defined through an HPC center's environment modules, it is recommended that you set this variable appropriately. + * :code:`NETCDF_C_ROOT`: This is the path to your NetCDF-C installation. Since this environment variable is not necessarily defined through an HPC center's environment modules, it is recommended that you set this variable appropriately. + * :code:`GPU_ARCH` : This is the AMD GPU architecture code for the target GPU you want to build for. This variable defaults to :code:`gfx90a`, which corresponds to the MI210, MI250, and MI250X GPUs. + +The :code:`FastEddy` executable will be located in the **SRC/FEMAIN** directory. To +build on other HPC systems with NVIDIA GPUs, check for availability of the aformentioned +modules/dependencies. Successful compilation may require modifications to shell environment +variable include or library paths, or alternatively minor adjustments to the include or library +flags in **SRC/FEMAIN/Makefile.hip**. From 7c626f003c21c52b491bc00d7705541878e3a45e Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Mon, 27 Jan 2025 14:55:52 -0700 Subject: [PATCH 3/7] Fixing link formatting --- docs/run_amd_gpus.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/run_amd_gpus.rst b/docs/run_amd_gpus.rst index 434293a..6ee1089 100644 --- a/docs/run_amd_gpus.rst +++ b/docs/run_amd_gpus.rst @@ -8,7 +8,7 @@ These instructions will help users get started with building FastEddy for system * `Oak Ridge National Laboratory's Frontier `_ * `Pawsey Supercomputing Centre's Setonix `_ - * `CSCS's Lumi _` + * `CSCS's Lumi `_ * `Fluid Numerics' Galapagos `_ From f42667d7bdfc576a981a5727a6d3b4ce2d2678c8 Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Wed, 19 Feb 2025 14:16:56 -0700 Subject: [PATCH 4/7] Feature 64 publication (#65) * Per #64, added new publication in Urban Climate * Per #64, fixed formatting --- docs/publications.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/publications.rst b/docs/publications.rst index 79aa6b8..e943cde 100644 --- a/docs/publications.rst +++ b/docs/publications.rst @@ -4,6 +4,14 @@ Publications ************ +.. _urban_climate_2025: + +| Munoz-Esparza, D., J. Sauer, P.A. Jimenez, J. Boehnert, D. Hahn, and M. Steiner. +| “Multiscale Weather Forecasting Sensitivities to Urban Characteristics and Atmospheric +| Conditions During a Cold Front Passage Over the Dallas-Fort Worth Metroplex”. +| *Urban Climate*, 60 (102334) 2025 +| https://doi.org/10.1016/j.uclim.2025.102334 + .. _wind_energy_2024: | Sanchez Gomez, M., D. Muñoz-Esparza, J.A. Sauer. "Implementation and Validation From d0ebaa7eafe372b498cb4b3f27898e5ec529334d Mon Sep 17 00:00:00 2001 From: Julie Prestopnik Date: Wed, 19 Feb 2025 14:17:29 -0700 Subject: [PATCH 5/7] Feature 50 contributing authors (#66) * Per #50, adding contributing authors * Per #50, fixing formatting of header by removing extraneous character --- docs/index.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/index.rst b/docs/index.rst index a2e7199..dedb148 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,6 +15,9 @@ and multiscale, turbulence-resolving, atmospheric boundary layer modeling into local scale weather prediction or actionable science and engineering applications. +Citations +--------- + The FastEddy code is located in an open, public `GitHub FastEddy-model repository `_. FastEddy should be cited as shown here: @@ -25,6 +28,17 @@ FastEddy should be cited as shown here: | https://doi.org/10.1029/2020MS002100 +Contributing Authors +-------------------- + +The following authors have contributed to this documentation of FastEddy: + +* Jeremy Sauer +* Domingo Muñoz-Esparza +* Eric Hendricks +* Julie Prestopnik + + .. toctree:: :hidden: From b6d5b55e7599810af06df656aacbd8a37d53a14f Mon Sep 17 00:00:00 2001 From: Jeff Hancock Date: Tue, 4 Mar 2025 14:36:05 -0700 Subject: [PATCH 6/7] feature branch with CIT workflow --- .github/workflows/CIT_derecho.yml | 53 +++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 .github/workflows/CIT_derecho.yml diff --git a/.github/workflows/CIT_derecho.yml b/.github/workflows/CIT_derecho.yml new file mode 100644 index 0000000..42883e7 --- /dev/null +++ b/.github/workflows/CIT_derecho.yml @@ -0,0 +1,53 @@ +# This is a basic workflow to help you get started with Actions + +name: CIT on derecho + +# Controls when the workflow will run +on: + # Triggers the workflow on pull request events but only for the "feature_CIT" branch + pull_request: + branches: + - feature_CIT + paths-ignore: + - 'docs/**' + - '.github/pull_request_template.md' + - '.github/ISSUE_TEMPLATE/**' + - '**/README.md' + - '**/LICENSE.md' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + build: + # Only do the actions when merged or manually run + if: github.event.label.name == 'run-pr-tests' || github.event_name == 'workflow_dispatch' + # The type of runner that the job will run on + #runs-on: ubuntu-latest + runs-on: shr-derecho + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it v2 or v4? (Julie's example shows v2) + - uses: actions/checkout@v4 + + # Runs a single command using the runners shell + - name: Run a one-line script + run: echo Hello, world! + + # Compile the FastEddy code + #- name: Compile FastEddy + # run: | + # cd SRC/FEMAIN + # module load cuda + # make clean + # make + + # Runs a set of commands using the runners shell + #- name: Run a multi-line script + # run: | + # echo Append datetime to a file + # /glade/work/fasteddyrt/actions-runner/appendDatetimeRun.sh + \ No newline at end of file From 881b8ec1092d0e11ccdb47517e3a79e3e1580325 Mon Sep 17 00:00:00 2001 From: Jeff Hancock Date: Mon, 10 Mar 2025 15:17:40 -0600 Subject: [PATCH 7/7] bogus change to test triggering --- docs/BOGUS.txt | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/BOGUS.txt diff --git a/docs/BOGUS.txt b/docs/BOGUS.txt new file mode 100644 index 0000000..6497c21 --- /dev/null +++ b/docs/BOGUS.txt @@ -0,0 +1,2 @@ +Just a bogus document to provide a change with which to try a pull request. Trying to +see if I can make it trigger the workflow! \ No newline at end of file