Scores by Region (#75)

* add region averages

* add regions to config

* Add regions to verification module, scripts, and rules

* add stratification to forecaster config and fix typo

* fix dict indexing

* fix append error

* read lon/lat from obs dataset

* Add inner verification domain

* Add missing dependency

* add plots by region

* Add regions to dashboard

* Fix dashboard

* Add region name and initializations to plot title (and remove header div)

* Add support for multiple regions

* Fix legend

Author: jonasbhend

config/forecasters-co1e.yaml

@@ -29,6 +29,16 @@ analysis:
   label: COSMO KENDA
   analysis_zarr: /scratch/mch/fzanetta/data/anemoi/datasets/mch-co1e-an-archive-0p01-2019-2024-1h-v1-pl13.zarr
 
+stratification:
+  regions:
+    - jura
+    - mittelland
+    - voralpen
+    - alpennordhang
+    - innerealpentaeler
+    - alpensuedseite
+  root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517
+
 locations:
   output_root: output/
   mlflow_uri:

config/forecasters.yaml

@@ -24,6 +24,16 @@ analysis:
   label: COSMO KENDA
   analysis_zarr: /scratch/mch/fzanetta/data/anemoi/datasets/mch-co2-an-archive-0p02-2015-2020-6h-v3-pl13.zarr
 
+stratification:
+  regions:
+    - jura
+    - mittelland
+    - voralpen
+    - alpennordhang
+    - innerealpentaeler
+    - alpensuedseite
+  root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517
+
 locations:
   output_root: output/
   mlflow_uri:

config/interpolators.yaml

@@ -48,6 +48,16 @@ analysis:
   label: COSMO KENDA
   analysis_zarr: /scratch/mch/fzanetta/data/anemoi/datasets/mch-co2-an-archive-0p02-2015-2020-1h-v3-pl13.zarr
 
+stratification:
+  regions:
+    - jura
+    - mittelland
+    - voralpen
+    - alpennordhang
+    - innerealpentaeler
+    - alpensuedseite
+  root: /scratch/mch/bhendj/regions/Prognoseregionen_LV95_20220517
+
 locations:
   output_root: output/
   mlflow_uri:

pyproject.toml

@@ -18,6 +18,9 @@ dependencies = [
     "pydantic>=2.11.7",
     "toml>=0.10.2",
     "netcdf4>=1.7.2",
+    "shapely>=2.1.2",
+    "cartopy>=0.25.0",
+    "pyproj>=3.7.2",
 ]
 
 [project.optional-dependencies]

resources/report/dashboard/script.js

@@ -12,6 +12,16 @@ document.querySelectorAll(".tab-link").forEach(button => {
 // Initialize selection widgets
 const choicesInstances = {};
 
+choicesInstances["region-select"] = new Choices("#region-select", {
+  searchEnabled: false,
+  removeItemButton: true,
+  shouldSort: false,
+  itemSelectText: "",
+  placeholder: false
+});
+document.getElementById("region-select").addEventListener("change", updateChart);
+
+
 choicesInstances["source-select"] = new Choices("#source-select", {
   searchEnabled: false,
   removeItemButton: true,
@@ -41,57 +51,74 @@ document.getElementById("param-select").addEventListener("change", updateChart);
 
 // Get the data (embedded in the HTML)
 data = JSON.parse(document.getElementById("verif-data").textContent)
+header = document.getElementById("header-text").textContent.trim()
 
 // Define base spec
 var spec = {
   "data": { "values": data },
-  "params": [
-    {
-      "name": "xZoom",
-      "select": {
-        "type": "interval",
-        "encodings": ["x"],
-        "zoom": "wheel![!event.shiftKey]"
-      },
-      "bind": "scales"
-    }
-  ],
+  "config": {
+    "scale": { "continuousPadding": 1 }
+  },
   "facet": {
-    "column": { "field": "param" },
-    "row": { "field": "metric" }
+    "row": { "field": "metric", "type": "nominal", "title": null },
+    "column": { "field": "param", "type": "nominal" , "title": null },
+  },
+  "resolve": {
+    "scale": {
+      "x": "shared",
+      "y": "independent"
+    },
   },
   "spec": {
-    "mark": "line",
+    "params": [
+      {
+        "name": "xZoom",
+        "select": {
+          "type": "interval",
+          "encodings": ["x"],
+          "zoom": "wheel![!event.shiftKey]"
+        },
+        "bind": "scales"
+      }
+    ],
+    "mark": {"type": "line", "point": { "size": 50 } },
     "width": 300,
     "height": 200,
     "encoding": {
       "x": {
         "field": "lead_time",
-        "type": "quantitative",
-        "axis": { "labels": true, "ticks": true },
+        "type": "quantitative"
       },
       "y": {
         "field": "value",
-        "type": "quantitative",
-        "scale": { "zero": false }
+        "type": "quantitative" ,
+          "scale": { "zero": false }
       },
       "color": {
         "field": "source",
-        "legend": { "orient": "top", "labelLimit": 1000, "symbolSize": 1000 }
-      }
+        "type": "nominal",
+        "legend": { "orient": "top", "title": "Data Source", "offset": 0, "padding": 10 }
+      },
+      "shape": {
+        "field": "region",
+        "type": "nominal",
+        "legend": { "orient": "top", "title": "Region", "offset": 0, "padding": 10 }
+      },
+      "strokeDash": {
+        "field": "region",
+        "type": "nominal",
+        "legend": null
+      },
+      "tooltip": [
+        { "field": "region", "type": "nominal", "title": "Region" },
+        { "field": "source", "type": "nominal", "title": "Source" },
+        { "field": "param", "type": "nominal", "title": "Parameter" },
+        { "field": "metric", "type": "nominal", "title": "Metric" },
+        { "field": "lead_time", "type": "quantitative", "title": "Lead Time (h)" },
+        { "field": "value", "type": "quantitative", "title": "Value" }
+      ]
     },
-    "transform": [
-      {
-        "filter": { "param": "xZoom" }
-      }
-    ]
   },
-  "resolve": {
-    "scale": {
-      "x": "shared",
-      "y": "independent"
-    }
-  }
 };
 
 
@@ -102,13 +129,18 @@ function getSelectedValues(id) {
 }
 
 function updateChart() {
+  const selectedRegions = getSelectedValues("region-select");
   const selectedSources = getSelectedValues("source-select");
   const selectedparams = getSelectedValues("param-select");
   const selectedMetrics = getSelectedValues("metric-select");
 
   const newSpec = JSON.parse(JSON.stringify(spec));
   const filters = [];
 
+  newSpec.title = "Verification using " + header;
+  if (selectedRegions.length > 0) {
+    filters.push({ field: "region", oneOf: selectedRegions });
+  }
   if (selectedSources.length > 0) {
     filters.push({ field: "source", oneOf: selectedSources });
   }

resources/report/dashboard/template.html.jinja2

@@ -101,12 +101,15 @@
 
     <!-- First tab -->
     <div id="tab1" class="tab-content active">
-        <div class="header">
-            <p>
-            {{header_text}}
-            </p>
-        </div>
         <div class="controls">
+            <div class="control-group">
+                <label>Region</label>
+                <select id="region-select" multiple>
+                    {% for region in regions %}
+                    <option value="{{region}}"{% if region == 'all' %} selected{% endif %}>{{region}}</option>
+                    {% endfor %}
+                </select>
+            </div>
             <div class="control-group">
                 <label>Source(s)</label>
                 <select id="source-select" multiple>
@@ -144,6 +147,10 @@
         {{ verif_data | safe | indent(8, false)}}
     </script>
 
+    <script id="header-text" type="text">
+        {{ header_text }}
+    </script>
+
     <script>
         {{ js_src | indent(8, true) }}
     </script>

src/evalml/config.py

@@ -205,6 +205,19 @@ class Locations(BaseModel):
     )
 
 
+class Stratification(BaseModel):
+    """Stratification settings for the analysis."""
+
+    regions: List[str] = Field(
+        ...,
+        description="List of region names for stratification.",
+    )
+    root: str = Field(
+        ...,
+        description="Root directory where the region shapefiles are stored.",
+    )
+
+
 class DefaultResources(BaseModel):
     """Default resource settings for job execution."""
 
@@ -276,6 +289,7 @@ class ConfigModel(BaseModel):
         description="Dictionary of baselines to include in the verification.",
     )
     analysis: AnalysisConfig
+    stratification: Stratification
     locations: Locations
     profile: Profile