Merge pull request #2256 from CliMA/gb/force_synchronize

Sbozzolo · web-flow · commit 9f4a732e5e2a · 2025-03-27T16:12:15.000-07:00
Sync CUDA streams before MPI calls
diff --git a/NEWS.md b/NEWS.md
@@ -4,6 +4,14 @@ ClimaCore.jl Release Notes
 main
 -------
 
+
+v0.14.29
+-------
+
+ - Fix empty interpolation with certain versions of CUDA+MPI
+   [2256](https://github.com/CliMA/ClimaCore.jl/pull/2256).
+
+
 v0.14.28
 -------
 
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "ClimaCore"
 uuid = "d414da3d-4745-48bb-8d80-42e94e092884"
 authors = ["CliMA Contributors <clima-software@caltech.edu>"]
-version = "0.14.28"
+version = "0.14.29"
 
 [deps]
 Adapt = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
diff --git a/src/Remapping/Remapping.jl b/src/Remapping/Remapping.jl
@@ -15,6 +15,7 @@ import ..DataLayouts,
     ..Fields,
     ..Hypsography
 import ClimaCore.Utilities: half
+import ClimaCore.Spaces: cuda_synchronize
 
 using ..RecursiveApply
 
diff --git a/src/Remapping/distributed_remapping.jl b/src/Remapping/distributed_remapping.jl
@@ -751,6 +751,7 @@ function _collect_interpolated_values!(
     index_field_end::Int;
     only_one_field,
 )
+    cuda_synchronize(ClimaComms.device(remapper.comms_ctx))   # Sync streams before MPI calls
     if only_one_field
         ClimaComms.reduce!(
             remapper.comms_ctx,

Original file line number	Diff line number	Diff line change
`@@ -751,6 +751,7 @@ function _collect_interpolated_values!(`
`751`	`751`	`index_field_end::Int;`
`752`	`752`	`only_one_field,`
`753`	`753`	`)`
	`754`	`+ cuda_synchronize(ClimaComms.device(remapper.comms_ctx)) # Sync streams before MPI calls`
`754`	`755`	`if only_one_field`
`755`	`756`	`ClimaComms.reduce!(`
`756`	`757`	`remapper.comms_ctx,`