Merge pull request #3632 from CliMA/sk/add_ss_to_cpu_scaling_pipeline

akshaysridhar · web-flow · commit 2c97b68977ac · 2025-06-18T20:28:57.000Z
Add MBW strong scaling runs to CPU scaling pipeline.
diff --git a/.buildkite/cpu_scaling_pipeline/README.md b/.buildkite/cpu_scaling_pipeline/README.md
@@ -0,0 +1,15 @@
+# Instructions to trigger CPU scaling build
+
+1). Visit url: https://buildkite.com/clima/climaatmos-dot-jl-cpu-scaling
+
+2). Click on "New Build"
+
+3). Set branch to the PR branch
+
+4). Click on "Create Build"
+
+The CPU scaling build is useful for understanding the CPU scaling performance implications of merging a PR.
+For higher-resolution simulations, with horizontal resolutions of 13 km or finer, memory footprint constraints 
+dictated use of at most (16) message passing interface (MPI) ranks per node. To maintain consistency
+in the CPU scaling studies, we use (16) MPI ranks per node for all simulations, although this results in 
+under-utilization of the available computing resources at lower resolutions.
diff --git a/.buildkite/cpu_scaling_pipeline/generate_pipeline.jl b/.buildkite/cpu_scaling_pipeline/generate_pipeline.jl
@@ -2,8 +2,16 @@
 # To run: `julia --project=.buildkite .buildkite/cpu_scaling_pipeline/generate_pipeline.jl`
 # nodes = (1, 2, 4, 8, 16, 32)
 # helems = (30, 42, 60, 84, 120, 170)
-nodes = (1, 2, 4)
-helems = (30, 42, 60)
+
+#strong scaling
+ss_nodes = (1, 2, 4) # number of nodes for weak scaling runs
+ss_helems = (30, 60, 120) # helems for weak scaling runs
+ss_procspernode = 16 # number of MPI processes per node
+
+# weak scaling
+ws_nodes = (1, 2, 4) # number of nodes for weak scaling runs
+ws_helems = (30, 42, 60) # helems for weak scaling runs
+ws_procspernode = 16 # number of MPI processes per node
 
 import YAML
 
@@ -36,19 +44,41 @@ init_step = Dict(
     ),
 )
 
-function generate_step(nodes::Int, helems::Int)
+function generate_step_ws(nodes::Int, helems::Int, procspernode::Int)
     return Dict(
-        "label" => ":computer: $nodes node, 16 processes per node, helem = $helems",
+        "label" => ":computer: MBW weak scaling, $nodes nodes, $procspernode processes per node, helem = $helems",
         "command" => "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $MBW_SCALING_CONFIG_PATH/moist_baroclinic_wave_helem_$(helems)_0M_ws.yml --job_id moist_baroclinic_wave_helem_$(helems)_0M_ws",
         "artifact_paths" => "moist_baroclinic_wave_helem_$(helems)_0M_ws/output_active/*",
+        "key" => "ws_$(nodes)_nodes",
         "agents" => Dict(
             "slurm_constraint" => CPU_TYPE,
             "queue" => "new-central",
             "slurm_nodes" => nodes,
-            "slurm_tasks_per_node" => 16,
+            "slurm_tasks_per_node" => procspernode,
             "slurm_cpus_per_task" => 1,
             "slurm_mem" => 0,
             "slurm_time" => "1:00:00",
+            "slurm_reservation" => "false",
+            "slurm_exclusive" => true,
+        ),
+    )
+end
+
+function generate_step_ss(nodes::Int, helems::Int, procspernode::Int)
+    return Dict(
+        "label" => ":computer: MBW strong scaling, $nodes nodes, $procspernode processes per node, helem = $helems",
+        "command" => "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file $MBW_SCALING_CONFIG_PATH/moist_baroclinic_wave_helem_$(helems)_0M_ss.yml --job_id moist_baroclinic_wave_helem_$(helems)_0M_ss",
+        "artifact_paths" => "moist_baroclinic_wave_helem_$(helems)_0M_ss/output_active/*",
+        "key" => "ss_$(nodes)_nodes",
+        "agents" => Dict(
+            "slurm_constraint" => CPU_TYPE,
+            "queue" => "new-central",
+            "slurm_nodes" => nodes,
+            "slurm_tasks_per_node" => procspernode,
+            "slurm_cpus_per_task" => 1,
+            "slurm_mem" => 0,
+            "slurm_time" => "1:00:00",
+            "slurm_reservation" => "false",
             "slurm_exclusive" => true,
         ),
     )
@@ -62,7 +92,15 @@ pipeline = Dict(
         "wait",
         Dict(
             "group" => "Moist Baroclinic Wave, weak scaling",
-            "steps" => [generate_step.(nodes, helems)...],
+            "steps" => [
+                generate_step_ws.(ws_nodes, ws_helems, ws_procspernode)...,
+            ],
+        ),
+        Dict(
+            "group" => "Moist Baroclinic Wave, strong scaling",
+            "steps" => [
+                generate_step_ss.(ss_nodes, ss_helems, ss_procspernode)...,
+            ],
         ),
     ],
 )
diff --git a/.buildkite/cpu_scaling_pipeline/pipeline.yml b/.buildkite/cpu_scaling_pipeline/pipeline.yml
@@ -18,7 +18,8 @@ steps:
       JULIA_MAX_NUM_PRECOMPILE_FILES: 50
   - "wait"
   - steps:
-      - label: ":computer: 1 node, 16 processes per node, helem = 30"
+      - label: ":computer: MBW weak scaling, 1 nodes, 16 processes per node, helem = 30"
+        key: "ws_1_nodes"
         agents:
           slurm_time: "1:00:00"
           slurm_cpus_per_task: 1
@@ -28,9 +29,11 @@ steps:
           slurm_exclusive: true
           slurm_mem: 0
           slurm_constraint: "icelake"
+          slurm_reservation: "false"
         command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_30_0M_ws.yml --job_id moist_baroclinic_wave_helem_30_0M_ws"
         artifact_paths: "moist_baroclinic_wave_helem_30_0M_ws/output_active/*"
-      - label: ":computer: 2 node, 16 processes per node, helem = 42"
+      - label: ":computer: MBW weak scaling, 2 nodes, 16 processes per node, helem = 42"
+        key: "ws_2_nodes"
         agents:
           slurm_time: "1:00:00"
           slurm_cpus_per_task: 1
@@ -40,9 +43,11 @@ steps:
           slurm_exclusive: true
           slurm_mem: 0
           slurm_constraint: "icelake"
+          slurm_reservation: "false"
         command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_42_0M_ws.yml --job_id moist_baroclinic_wave_helem_42_0M_ws"
         artifact_paths: "moist_baroclinic_wave_helem_42_0M_ws/output_active/*"
-      - label: ":computer: 4 node, 16 processes per node, helem = 60"
+      - label: ":computer: MBW weak scaling, 4 nodes, 16 processes per node, helem = 60"
+        key: "ws_4_nodes"
         agents:
           slurm_time: "1:00:00"
           slurm_cpus_per_task: 1
@@ -52,9 +57,54 @@ steps:
           slurm_exclusive: true
           slurm_mem: 0
           slurm_constraint: "icelake"
+          slurm_reservation: "false"
         command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_60_0M_ws.yml --job_id moist_baroclinic_wave_helem_60_0M_ws"
         artifact_paths: "moist_baroclinic_wave_helem_60_0M_ws/output_active/*"
     group: "Moist Baroclinic Wave, weak scaling"
+  - steps:
+      - label: ":computer: MBW strong scaling, 1 nodes, 16 processes per node, helem = 30"
+        key: "ss_1_nodes"
+        agents:
+          slurm_time: "1:00:00"
+          slurm_cpus_per_task: 1
+          slurm_tasks_per_node: 16
+          queue: "new-central"
+          slurm_nodes: 1
+          slurm_exclusive: true
+          slurm_mem: 0
+          slurm_constraint: "icelake"
+          slurm_reservation: "false"
+        command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_30_0M_ss.yml --job_id moist_baroclinic_wave_helem_30_0M_ss"
+        artifact_paths: "moist_baroclinic_wave_helem_30_0M_ss/output_active/*"
+      - label: ":computer: MBW strong scaling, 2 nodes, 16 processes per node, helem = 60"
+        key: "ss_2_nodes"
+        agents:
+          slurm_time: "1:00:00"
+          slurm_cpus_per_task: 1
+          slurm_tasks_per_node: 16
+          queue: "new-central"
+          slurm_nodes: 2
+          slurm_exclusive: true
+          slurm_mem: 0
+          slurm_constraint: "icelake"
+          slurm_reservation: "false"
+        command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_60_0M_ss.yml --job_id moist_baroclinic_wave_helem_60_0M_ss"
+        artifact_paths: "moist_baroclinic_wave_helem_60_0M_ss/output_active/*"
+      - label: ":computer: MBW strong scaling, 4 nodes, 16 processes per node, helem = 120"
+        key: "ss_4_nodes"
+        agents:
+          slurm_time: "1:00:00"
+          slurm_cpus_per_task: 1
+          slurm_tasks_per_node: 16
+          queue: "new-central"
+          slurm_nodes: 4
+          slurm_exclusive: true
+          slurm_mem: 0
+          slurm_constraint: "icelake"
+          slurm_reservation: "false"
+        command: "srun julia --color=yes --project=.buildkite .buildkite/ci_driver.jl --config_file config/mbw_scaling_configs/moist_baroclinic_wave_helem_120_0M_ss.yml --job_id moist_baroclinic_wave_helem_120_0M_ss"
+        artifact_paths: "moist_baroclinic_wave_helem_120_0M_ss/output_active/*"
+    group: "Moist Baroclinic Wave, strong scaling"
 env:
   SLURM_KILL_BAD_EXIT: 1
   OPENBLAS_NUM_THREADS: 1
