Skip to content

Commit 61b871e

Browse files
dnerinidnerini
committed
Merge branch 'main' into plotting
2 parents 421b478 + 67b1906 commit 61b871e

File tree

16 files changed

+108
-115
lines changed

16 files changed

+108
-115
lines changed

‎README.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,25 +31,25 @@ description: |
3131
dates:
3232
start: 2020-01-01T12:00
3333
end: 2020-01-10T00:00
34-
frequency: 54h
34+
frequency: 60h
3535

3636
runs:
3737
- forecaster:
3838
mlflow_id: 2f962c89ff644ca7940072fa9cd088ec
3939
label: Stage D - N320 global grid with CERRA finetuning
40-
steps: 0/126/6
40+
steps: 0/120/6
4141
- forecaster:
4242
mlflow_id: d0846032fc7248a58b089cbe8fa4c511
4343
label: M-1 forecaster
44-
steps: 0/126/6
44+
steps: 0/120/6
4545

4646

4747
baselines:
4848
- baseline:
4949
baseline_id: COSMO-E
5050
label: COSMO-E
5151
root: /store_new/mch/msopr/ml/COSMO-E
52-
steps: 0/126/6
52+
steps: 0/120/6
5353

5454
analysis:
5555
label: COSMO KENDA

‎config/forecasters-co1e.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ description: |
44
(KENDA-1) at 1km resolution.
55
66
dates:
7-
start: 2024-01-01T12:00
8-
end: 2024-01-10T00:00
7+
start: 2020-08-01T12:00
8+
end: 2020-08-10T00:00
99
frequency: 54h
1010

1111
runs:
1212
- forecaster:
1313
mlflow_id: 2174c939c8844555a52843b71219d425
1414
label: Cosmo 1km + era5 N320, finetuned on cerra checkpoint, lam resolution 11
15-
config: resources/inference/configs/forecaster_no_trimedge_with_global.yaml
16-
steps: 0/126/6
15+
config: resources/inference/configs/forecaster_no_trimedge_fromtraining.yaml
16+
steps: 0/120/6
1717
inference_resources:
1818
gpu: 4
1919
tasks: 4
@@ -25,7 +25,7 @@ baselines:
2525
baseline_id: COSMO-1E
2626
label: COSMO-1E
2727
root: /scratch/mch/bhendj/COSMO-1E
28-
steps: 0/126/6
28+
steps: 0/33/6
2929

3030
analysis:
3131
label: COSMO KENDA

‎config/forecasters.yaml‎

Lines changed: 5 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,24 @@
11
# yaml-language-server: $schema=../workflow/tools/config.schema.json
22
description: |
3-
This is an experiment to do blabla.
3+
Evaluate skill of COSMO-E emulator (M-1 forecaster).
44
55
dates:
66
start: 2020-01-01T12:00
77
end: 2020-01-10T00:00
8-
frequency: 36h
8+
frequency: 60h
99

1010
runs:
11-
- forecaster:
12-
mlflow_id: 2f962c89ff644ca7940072fa9cd088ec
13-
label: Stage D - N320 global grid with CERRA finetuning
14-
config: resources/inference/configs/forecaster_with_global.yaml
15-
steps: 0/126/6
1611
- forecaster:
1712
mlflow_id: d0846032fc7248a58b089cbe8fa4c511
1813
label: M-1 forecaster
19-
config: resources/inference/configs/forecaster_with_global.yaml
20-
steps: 0/126/6
14+
steps: 0/120/6
2115

2216
baselines:
2317
- baseline:
2418
baseline_id: COSMO-E
2519
label: COSMO-E
2620
root: /store_new/mch/msopr/ml/COSMO-E
27-
steps: 0/126/6
21+
steps: 0/120/6
2822

2923
analysis:
3024
label: COSMO KENDA
@@ -39,7 +33,7 @@ locations:
3933
profile:
4034
executor: slurm
4135
global_resources:
42-
gpus: 15
36+
gpus: 16
4337
default_resources:
4438
slurm_partition: "postproc"
4539
cpus_per_task: 1

‎config/interpolators.yaml‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
# yaml-language-server: $schema=../workflow/tools/config.schema.json
22
description: |
3-
Stretched interpolator vs LAM interpolator.
3+
Evaluate skill of SGM interpolator (M-2 interpolator).
44
55
dates:
66
start: 2020-01-01T12:00
77
end: 2020-01-10T00:00
8-
frequency: 54h
8+
frequency: 60h
99

1010
runs:
1111
- interpolator:
12-
mlflow_id: 9c18b90074214d769b8b383722fc5a06
12+
mlflow_id: 8d1e0410ca7d4f74b368b3079878259a
1313
label: M-2 interpolator (KENDA)
14-
steps: 0/121/1
15-
config: resources/inference/configs/interpolator_from_test_data.yaml
14+
steps: 0/120/1
15+
config: resources/inference/configs/interpolator_from_test_data_stretched.yaml
1616
forecaster: null
1717
extra_dependencies:
1818
- git+https://github.com/ecmwf/anemoi-inference@14189907b4f4e3b204b7994f828831b8aa51e9b6
@@ -21,12 +21,12 @@ runs:
2121
- interpolator:
2222
mlflow_id: 8d1e0410ca7d4f74b368b3079878259a
2323
label: M-2 interpolator (M-1 forecaster)
24-
steps: 0/121/1
24+
steps: 0/120/1
2525
config: resources/inference/configs/interpolator_stretched.yaml
2626
forecaster:
2727
mlflow_id: d0846032fc7248a58b089cbe8fa4c511
2828
config: resources/inference/configs/forecaster_with_global.yaml
29-
steps: 0/126/6
29+
steps: 0/120/6
3030
extra_dependencies:
3131
- git+https://github.com/ecmwf/anemoi-inference@14189907b4f4e3b204b7994f828831b8aa51e9b6
3232
- torch-geometric==2.6.1
@@ -35,14 +35,14 @@ runs:
3535
mlflow_id: d0846032fc7248a58b089cbe8fa4c511
3636
label: M-1 forecaster
3737
config: resources/inference/configs/forecaster_with_global.yaml
38-
steps: 0/126/6
38+
steps: 0/120/6
3939

4040
baselines:
4141
- baseline:
4242
baseline_id: COSMO-E-1h
4343
label: COSMO-E
4444
root: /scratch/mch/bhendj/COSMO-E
45-
steps: 0/121/1
45+
steps: 0/120/1
4646

4747
analysis:
4848
label: COSMO KENDA
@@ -58,7 +58,7 @@ locations:
5858
profile:
5959
executor: slurm
6060
global_resources:
61-
gpus: 15
61+
gpus: 16
6262
default_resources:
6363
slurm_partition: "postproc"
6464
cpus_per_task: 1

‎config/showcase.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,14 @@ runs:
1212
mlflow_id: d0846032fc7248a58b089cbe8fa4c511
1313
label: M-1 forecaster
1414
config: resources/inference/configs/forecaster_with_global.yaml
15-
steps: 0/126/6
15+
steps: 0/120/6
1616

1717
baselines:
1818
- baseline:
1919
baseline_id: COSMO-E
2020
label: COSMO-E
2121
root: /store_new/mch/msopr/ml/COSMO-E
22-
steps: 0/12/6
22+
steps: 0/120/6
2323

2424
analysis:
2525
label: COSMO KENDA

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,4 +53,4 @@ markers = [
5353
packages = [
5454
"src/evalml",
5555
"src/verification"
56-
]
56+
]

‎src/evalml/config.py‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,8 +74,10 @@ class RunConfig(BaseModel):
7474
...,
7575
description=(
7676
"Forecast lead times in hours, formatted as 'start/end/step'. "
77-
"The range is half-open [start, end), meaning it includes the start "
78-
"but excludes the end. Example: '0/126/6' for lead times every 6 hours up to 120 hours."
77+
"The range includes the start lead time and continues with the given step "
78+
"until reaching or exceeding the end lead time. "
79+
"Example: '0/120/6' for lead times every 6 hours up to 120 h, "
80+
"or '0/33/6' up to 30 h."
7981
),
8082
)
8183
extra_dependencies: List[str] = Field(
@@ -102,15 +104,13 @@ def validate_steps(cls, v: str) -> str:
102104
try:
103105
start, end, step = map(int, parts)
104106
except ValueError:
105-
raise ValueError("Start, end, and interval must be integers.")
106-
if start >= end:
107-
raise ValueError(f"Start ({start}) must be less than end ({end}).")
108-
if step <= 0:
109-
raise ValueError(f"Interval ({step}) must be a positive integer.")
110-
if (end - start) % step != 0:
107+
raise ValueError("Start, end, and step must be integers.")
108+
if start > end:
111109
raise ValueError(
112-
f"The step ({step}) must evenly divide the range ({end - start})."
110+
f"Start ({start}) must be less than or equal to end ({end})."
113111
)
112+
if step <= 0:
113+
raise ValueError(f"Step ({step}) must be a positive integer.")
114114
return v
115115

116116

‎workflow/Snakefile‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ rule sandbox_all:
6363
input:
6464
expand(
6565
rules.create_inference_sandbox.output.sandbox,
66-
run_id=collect_all_runs(),
66+
run_id=collect_all_candidates(),
6767
),
6868

6969

@@ -73,7 +73,7 @@ rule run_inference_all:
7373
expand(
7474
OUT_ROOT / "data/runs/{run_id}/{init_time}/raw",
7575
init_time=[t.strftime("%Y%m%d%H%M") for t in REFTIMES],
76-
run_id=collect_all_runs(),
76+
run_id=collect_all_candidates(),
7777
),
7878

7979

@@ -82,7 +82,7 @@ rule verif_metrics_all:
8282
expand(
8383
rules.verif_metrics.output,
8484
init_time=[t.strftime("%Y%m%d%H%M") for t in REFTIMES],
85-
run_id=collect_all_runs(),
85+
run_id=collect_all_candidates(),
8686
),
8787

8888

‎workflow/rules/common.smk‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,13 @@ REFTIMES = _reftimes()
6868

6969

7070
def collect_all_runs():
71-
"""Collect all runs defined in the configuration."""
71+
"""Collect all runs defined in the configuration, including secondary runs."""
7272
runs = {}
7373
for run_entry in copy.deepcopy(config["runs"]):
7474
model_type = next(iter(run_entry))
7575
run_config = run_entry[model_type]
7676
run_config["model_type"] = model_type
77+
run_config["is_candidate"] = True
7778
run_id = run_config["mlflow_id"][0:9]
7879

7980
if model_type == "interpolator":
@@ -84,6 +85,7 @@ def collect_all_runs():
8485
# Ensure a proper 'forecaster' entry exists with model_type
8586
fore_cfg = copy.deepcopy(run_config["forecaster"])
8687
fore_cfg["model_type"] = "forecaster"
88+
fore_cfg["is_candidate"] = False # exclude from outputs
8789
runs[tail_id] = fore_cfg
8890
run_id = f"{run_id}-{tail_id}"
8991

@@ -92,6 +94,16 @@ def collect_all_runs():
9294
return runs
9395

9496

97+
def collect_all_candidates():
98+
"""Collect participating runs ('candidates') only."""
99+
runs = collect_all_runs()
100+
candidates = {}
101+
for run_id, run_config in runs.items():
102+
if run_config.get("is_candidate", False):
103+
candidates[run_id] = run_config
104+
return candidates
105+
106+
95107
def collect_all_baselines():
96108
"""Collect all baselines defined in the configuration."""
97109
baselines = {}
@@ -108,7 +120,8 @@ def collect_experiment_participants():
108120
for base in BASELINE_CONFIGS.keys():
109121
participants[base] = OUT_ROOT / f"data/baselines/{base}/verif_aggregated.nc"
110122
for exp in RUN_CONFIGS.keys():
111-
participants[exp] = OUT_ROOT / f"data/runs/{exp}/verif_aggregated.nc"
123+
if RUN_CONFIGS[exp].get("is_candidate", False):
124+
participants[exp] = OUT_ROOT / f"data/runs/{exp}/verif_aggregated.nc"
112125
return participants
113126

114127

‎workflow/rules/data.smk‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,15 @@ if "extract_cosmoe" in config.get("include-optional-rules", []):
1818
runtime="24h",
1919
params:
2020
year_postfix=lambda wc: f"FCST{wc.year}",
21-
lead_time="0/126/6",
21+
steps="0/120/6",
2222
log:
2323
OUT_ROOT / "logs/extract-cosmoe-fcts-{year}.log",
2424
shell:
2525
"""
2626
python workflow/scripts/extract_baseline_fct.py \
2727
--archive_dir {input.archive}/{params.year_postfix} \
2828
--output_store {output.fcts} \
29-
--lead_time {params.lead_time} \
29+
--steps {params.steps} \
3030
> {log} 2>&1
3131
"""
3232

@@ -45,14 +45,14 @@ if "extract_cosmo1e" in config.get("include-optional-rules", []):
4545
runtime="24h",
4646
params:
4747
year_postfix=lambda wc: f"FCST{wc.year}",
48-
lead_time="0/34/1",
48+
steps="0/33/1",
4949
log:
5050
OUT_ROOT / "logs/extract-cosmo1e-fcts-{year}.log",
5151
shell:
5252
"""
5353
python workflow/scripts/extract_baseline_fct.py \
5454
--archive_dir {input.archive}/{params.year_postfix} \
5555
--output_store {output.fcts} \
56-
--lead_time {params.lead_time} \
56+
--steps {params.steps} \
5757
> {log} 2>&1
5858
"""

0 commit comments

Comments
 (0)