Use test env in buildkite

Author: charleskawczynski

.buildkite/pipeline.yml

@@ -20,14 +20,9 @@ steps:
           rm -rf ${JULIA_DEPOT_PATH}
         fi
 
-      - echo "--- Instantiate project"
-      - "julia --project -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
-      # force the initialization of the CUDA runtime as it is lazily loaded by default
-      - "julia --project -e 'using CUDA; CUDA.precompile_runtime()'"
-
       - echo "--- Instantiate test"
-      - "julia --project=test -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
-      - "julia --project=test --check-bounds=yes -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
+      - "julia --project=test --check-bounds=yes -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
+      - "julia --project=test -e 'using CUDA; CUDA.precompile_runtime()'"
 
       - echo "--- Instantiate lib/ClimaCoreTempestRemap"
       - "julia --project=lib/ClimaCoreTempestRemap -e 'using Pkg; Pkg.develop(path=\".\"); Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
@@ -43,7 +38,7 @@ steps:
       - "julia --project=benchmarks/bickleyjet -e 'using Pkg; Pkg.instantiate(;verbose=true); Pkg.precompile(;strict=true)'"
 
       - echo "--- Instantiate status"
-      - "julia --project -e 'using Pkg; Pkg.status()'"
+      - "julia --project=test -e 'using Pkg; Pkg.status()'"
 
     agents:
       slurm_mem: 80G
@@ -109,7 +104,7 @@ steps:
       - label: "Unit: data cuda"
         key: unit_data_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/DataLayouts/cuda.jl CUDA"
         agents:
           slurm_gpus: 1
@@ -181,15 +176,15 @@ steps:
       - label: "Unit: cuda spaces"
         key: "gpu_cuda_spaces"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/spaces.jl"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: distributed cuda spaces"
         key: "gpu_distributed_extruded_cuda_spaces"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/space_construction.jl CUDA"
         agents:
           slurm_gpus_per_task: 1
@@ -198,15 +193,15 @@ steps:
       - label: "Unit: ddss1"
         key: unit_ddss1
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Spaces/ddss1.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: ddss1 cs"
         key: unit_ddss1_cs
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Spaces/ddss1_cs.jl CUDA"
         agents:
           slurm_gpus: 1
@@ -303,7 +298,7 @@ steps:
       - label: "Unit: cuda dss 2-process test"
         key: "gpu_ddss2_test"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss2.jl"
         timeout_in_minutes: 15
         env:
@@ -315,7 +310,7 @@ steps:
       - label: "Unit: cuda dss 3-process test"
         key: "gpu_ddss3_test"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss3.jl"
         timeout_in_minutes: 15
         env:
@@ -327,7 +322,7 @@ steps:
       - label: "Unit: cuda dss 4-process test"
         key: "gpu_ddss4_test"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss4.jl"
         timeout_in_minutes: 15
         env:
@@ -339,7 +334,7 @@ steps:
       - label: "Unit: cuda Cubed Sphere dss; ne = 32; 2-process test"
         key: "gpu_ddss_ne32_cs_2processes"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss_ne32_cs.jl"
         timeout_in_minutes: 15
         env:
@@ -351,7 +346,7 @@ steps:
       - label: "Unit: cuda Cubed Sphere dss; ne = 32; 3-process test"
         key: "gpu_ddss_ne32_cs_3processes"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss_ne32_cs.jl"
         timeout_in_minutes: 15
         env:
@@ -363,7 +358,7 @@ steps:
       - label: "Unit: cuda Cubed Sphere dss; ne = 32; 4-process test"
         key: "gpu_ddss_ne32_cs_4processes"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Spaces/distributed_cuda/ddss_ne32_cs.jl"
         timeout_in_minutes: 15
         env:
@@ -382,23 +377,23 @@ steps:
       - label: "Unit: field cuda"
         key: unit_field_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Fields/field.jl"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: reduction cuda"
         key: unit_reduction_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Fields/reduction_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: distributed reduction cuda"
         key: unit_distributed_reduction_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "srun julia --color=yes --check-bounds=yes --project=test test/Fields/reduction_cuda_distributed.jl"
         agents:
           slurm_gpus_per_task: 1
@@ -516,31 +511,31 @@ steps:
         key: unit_rectilinear_cuda
         command: "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/rectilinear_cuda.jl"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/spectralelement/rectilinear_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: hybrid operators cuda"
         key: unit_ops_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: extruded sphere cuda"
         key: unit_extruded_sphere_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/extruded_sphere_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
 
       - label: "Unit: extruded 3dbox cuda"
         key: unit_extruded_3dbox_cuda
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --check-bounds=yes --project=test test/Operators/hybrid/extruded_3dbox_cuda.jl CUDA"
         agents:
           slurm_gpus: 1
@@ -947,7 +942,7 @@ steps:
       - label: "Unit: limiter cuda"
         key: unit_limiter_gpu
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --project=test test/Limiters/limiter.jl"
         agents:
           slurm_gpus: 1
@@ -995,7 +990,7 @@ steps:
       - label: "Perf: FD operators from the wild (gpu)"
         key: perf_FD_ops_examples_gpu
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --project=test test/Operators/finitedifference/opt_examples.jl"
         agents:
           slurm_gpus: 1
@@ -1020,7 +1015,7 @@ steps:
       - label: "Perf: SEM operator benchmarks (cuda Float32)"
         key: "perf_gpu_spectral_ops_cuda_float32"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA --float-type Float32"
         agents:
           slurm_gpus: 1
@@ -1032,7 +1027,7 @@ steps:
       - label: "Perf: SEM operator benchmarks (cuda Float64)"
         key: "perf_gpu_spectral_ops_cuda_float64"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA --float-type Float64"
         agents:
           slurm_gpus: 1
@@ -1048,7 +1043,7 @@ steps:
       - label: "Perf: SEM operator benchmarks"
         key: "perf_gpu_spectral_ops"
         command:
-          - "julia --project -e 'using CUDA; CUDA.versioninfo()'"
+          - "julia --project=test -e 'using CUDA; CUDA.versioninfo()'"
           - "julia --color=yes --project=test test/Operators/spectralelement/benchmark_ops.jl --device CUDA"
         agents:
           slurm_gpus: 1

ext/ClimaCoreCUDAExt.jl

@@ -2,7 +2,14 @@ module ClimaCoreCUDAExt
 
 import ClimaComms
 import ClimaCore: DataLayouts, Grids, Spaces, Fields
+import ClimaCore: Geometry
+import ClimaCore.Geometry: AxisTensor
 import CUDA
+using CUDA
+using CUDA: threadIdx, blockIdx, blockDim
+import StaticArrays: SVector, SMatrix, SArray
+import ClimaCore.RecursiveApply:
+    ⊠, ⊞, ⊟, radd, rmul, rsub, rdiv, rmap, rzero, rmin, rmax
 
 include(joinpath("cuda", "data_layouts.jl"))
 include(joinpath("cuda", "fields.jl"))

ext/cuda/fields.jl

@@ -3,8 +3,14 @@ using CUDA: @cuda
 import LinearAlgebra, Statistics
 import ClimaCore: DataLayouts, Spaces, Grids, Fields
 import ClimaCore.Fields: Field, FieldStyle
-import ClimaCore.Fields: AbstractFieldStyle
-import ClimaCore.Spaces: cuda_synchronize
+import ClimaCore.Fields: AbstractFieldStyle, bycolumn
+import ClimaCore.Spaces: AbstractSpace, cuda_synchronize
+
+function bycolumn(fn, space::AbstractSpace, ::ClimaComms.CUDADevice)
+    fn(:)
+    return nothing
+end
+
 function Base.sum(
     field::Union{Field, Base.Broadcast.Broadcasted{<:FieldStyle}},
     ::ClimaComms.CUDADevice,

ext/cuda/limiters.jl

@@ -1,5 +1,11 @@
-import ClimaCore.Limiters: QuasiMonotoneLimiter
+import ClimaCore.Limiters:
+    QuasiMonotoneLimiter,
+    compute_element_bounds!,
+    compute_neighbor_bounds_local!,
+    apply_limiter!
 import ClimaCore.Fields
+import ClimaCore: DataLayouts, Spaces, Topologies, Fields
+using CUDA
 
 function config_threadblock(Nv, Nh)
     nitems = Nv * Nh

ext/cuda/matrix_fields_multiple_field_solve.jl

@@ -2,6 +2,7 @@ import CUDA
 import ClimaComms
 import LinearAlgebra: UniformScaling
 import ClimaCore.Operators
+import ClimaCore.MatrixFields
 import ClimaCore.MatrixFields: multiple_field_solve!
 import ClimaCore.MatrixFields: is_CuArray_type
 import ClimaCore.MatrixFields: allow_scalar_func
@@ -13,9 +14,9 @@ is_CuArray_type(::Type{T}) where {T <: CUDA.CuArray} = true
 
 function multiple_field_solve!(::ClimaComms.CUDADevice, cache, x, A, b, x1)
     Ni, Nj, _, _, Nh = size(Fields.field_values(x1))
-    names = matrix_row_keys(keys(A))
+    names = MatrixFields.matrix_row_keys(keys(A))
     Nnames = length(names)
-    nthreads, nblocks = Topologies._configure_threadblock(Ni * Nj * Nh * Nnames)
+    nthreads, nblocks = _configure_threadblock(Ni * Nj * Nh * Nnames)
     sscache = Operators.strip_space(cache)
     ssx = Operators.strip_space(x)
     ssA = Operators.strip_space(A)

ext/cuda/matrix_fields_single_field_solve.jl

@@ -6,6 +6,7 @@ import ClimaCore.Fields: Field
 import ClimaCore.Fields
 import ClimaCore.MatrixFields: _single_field_solve!
 import ClimaCore.MatrixFields: band_matrix_solve!, unzip_tuple_field_values
+import ClimaCore.RecursiveApply: ⊠, ⊞, ⊟, rmap, rzero, rdiv
 
 # called by TuplesOfNTuples.jl's `inner_dispatch`:
 # which requires a particular argument order:

ext/cuda/operators_finite_difference.jl

@@ -3,7 +3,9 @@ import Base.Broadcast: Broadcasted
 import ClimaComms
 using CUDA: @cuda
 import ClimaCore.Operators: AbstractStencilStyle, strip_space
+import ClimaCore.Operators: setidx!, getidx
 import ClimaCore.Operators: StencilBroadcasted
+import ClimaCore.Operators: LeftBoundaryWindow, RightBoundaryWindow, Interior
 
 struct CUDAColumnStencilStyle <: AbstractStencilStyle end
 AbstractStencilStyle(::ClimaComms.CUDADevice) = CUDAColumnStencilStyle
@@ -24,11 +26,11 @@ function Base.copyto!(
         Nq = 1
         Nh = 1
     end
-    (li, lw, rw, ri) = bounds = window_bounds(space, bc)
+    (li, lw, rw, ri) = bounds = Operators.window_bounds(space, bc)
     Nv = ri - li + 1
     max_threads = 256
     nitems = Nv * Nq * Nq * Nh # # of independent items
-    (nthreads, nblocks) = Topologies._configure_threadblock(max_threads, nitems)
+    (nthreads, nblocks) = _configure_threadblock(max_threads, nitems)
     @cuda always_inline = true threads = (nthreads,) blocks = (nblocks,) copyto_stencil_kernel!(
         strip_space(out, space),
         strip_space(bc, space),
@@ -55,7 +57,13 @@ function copyto_stencil_kernel!(out, bc, space, bds, Nq, Nh, Nv)
                 RightBoundaryWindow{Spaces.right_boundary_name(space)}() :
                 Interior()
             )
-        setidx!(space, out, idx, hidx, getidx(space, bc, window, idx, hidx))
+        setidx!(
+            space,
+            out,
+            idx,
+            hidx,
+            Operators.getidx(space, bc, window, idx, hidx),
+        )
     end
     return nothing
 end