Merge branch 'master' of https://github.com/BSC-ES/providentia

Author: PaulaSerranoSierra

.github/workflows/pytest.yaml

@@ -22,10 +22,10 @@ jobs:
       run: conda env update --file environment.yaml
     - name: Test with pytest
       run: |
-        conda run -n providentia-env coverage run -m pytest tests
-        conda run -n providentia-env coverage report -i -m
+        conda run -n providentia-env_v3.0.0 coverage run -m pytest tests
+        conda run -n providentia-env_v3.0.0 coverage report -i -m
     - name: Generate coverage badge
-      run: conda run -n providentia-env coverage-badge > coverage.svg
+      run: conda run -n providentia-env_v3.0.0 coverage-badge > coverage.svg
     - name: Update coverage badge
       run: |
         git config --global user.name "github-actions[bot]"

.gitlab-ci.yml

@@ -13,4 +13,5 @@ report_filename = PROVIDENTIA_Report
 report_title = Report
 temporal_colocation = True
 observations_data_label = Nitrogen Dioxide Mass Concentration
+forecast = daily
 interpolated = false

configurations/actris_vre_cams.conf

@@ -0,0 +1,46 @@
+[All]
+network = eea/eionet
+species = pm10
+resolution = hourly
+start_date = 20240701
+end_date = 20250701
+experiments = a00c_cirrus2_pm-regional-000,a9dc2-regional-000 (Oper,OperDa)
+calibration_factor = a00c_cirrus2_pm-regional-000(*2),a9dc2-regional-000(*1)
+temporal_colocation = True
+spatial_colocation = True
+statistic_mode = Spatial|Temporal
+timeseries_statistic_aggregation=Mean
+report_type = debug
+report_summary = True
+harmonise_summary = False
+report_stations = False
+report_title = Providentia BDRC vs OperDa Report Eionet report
+report_filename = BDRC_vs_OperDA_eionet_filtered15_stations_dcal_a00c_150925
+
+[[Selected_stations]]
+    station_reference = keep: ES2024A,ES2044A,ES1767A,ES1768A,ES1920A,ES2002A,ES1955A,PT02004,PT02018,PT02022,PT02019,PT03102,PT03099,PT04002,PT05008,ES1878A,ES1781A,ES1593A,ES1751A,ES1750A,ES1370A,ES1549A,ES1996A,ES1921A,ES1885A,ES1825A,ES1831A,ES1604A,ES1827A,ES0296A,ES1518A,ES1093A,ES1517A,ES1828A,ES0694A,ES1275A,FR08714,FR08016,FR02043,FR03071,FR03069,FR03070,FR24009,FR24036,FR24023,FR41002,FR41001,IT1277A,IT1796A,IT2167A,SI0038A,RH0115,RH0108,MT00007 ||
+    map_extent = -25, 45, 20, 50
+
+[[Canaria]]
+    station_reference = keep: ES2024A,ES2044A,ES1767A,ES1768A,ES1920A,ES2002A,ES1955A ||
+    map_extent = -20, -10, 25, 32
+
+[[West Mediterrenean]]
+    station_reference = keep: PT02004,PT02018,PT02022,PT02019,PT03102,PT03099,PT04002,PT05008,ES1878A,ES1781A,ES1593A,ES1751A,ES1750A,ES1370A,ES1549A,ES1996A,ES1921A,ES1885A ||
+    map_extent = -20, 20, 35, 50
+
+[[Balears]]
+    station_reference = keep: ES1825A,ES1831A,ES1604A,ES1827A,ES0296A,ES1518A,ES1093A,ES1517A,ES1828A ||
+    map_extent = -5, 8, 38, 42
+
+[[Central Mediterrenean]]
+    station_reference = keep: ES0694A,ES1275A,FR08714,FR08016,FR02043,FR03071,FR03069,FR03070,FR24009,FR24036,FR24023,FR41002,FR41001,IT1277A ||
+    map_extent = -5, 20, 20, 45
+
+[[Eastern Mediterranean]]
+   station_reference = keep: IT1796A,IT2167A,SI0038A,RH0115,RH0108,MT00007 ||
+   map_extent = 0, 25, 30, 50
+
+[[Extra Mediterranean]]
+   station_reference = keep: GR0120A,BG0056A,BG0071A,BG0075A,CY0002R ||
+   map_extent = 10, 45, 25, 50

configurations/test_elina.conf

@@ -7,8 +7,9 @@ end_date = 20230201
 experiments = a59g-regional-000
 temporal_colocation = True
 spatial_colocation = True
-statistic_mode = Temporal|Spatial
+statistic_mode = Spatial|Temporal
 forecast = daily
 #forecast = combined
 #forecast = day
-report_type = operational
+report_type = operational
+interpolated = True

configurations/test_forecast.conf

@@ -53,7 +53,8 @@
 <li class="toctree-l3"><a class="reference internal" href="#starting-an-interpolation">Starting an Interpolation</a></li>
 <li class="toctree-l3"><a class="reference internal" href="#checking-output">Checking output</a></li>
 <li class="toctree-l3"><a class="reference internal" href="#define-experiments">Define experiments</a></li>
-<li class="toctree-l3"><a class="reference internal" href="#additional-considerations">Additional Considerations</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#additional-considerations">Additional considerations</a></li>
+<li class="toctree-l3"><a class="reference internal" href="#different-resolutions-between-model-and-observations">Different resolutions between model and observations</a></li>
 </ul>
 </li>
 <li class="toctree-l2"><a class="reference internal" href="Dashboard.html">Dashboard</a></li>
@@ -161,11 +162,15 @@ <h2>Checking output<a class="headerlink" href="#checking-output" title="Link to
 <p>If you are using a machine that allows both types of paths, it is recommended to list your <code class="docutils literal notranslate"><span class="pre">gpfs</span></code> paths first. This is because when reading data from the <code class="docutils literal notranslate"><span class="pre">esarchive</span></code>, a major limitation on the read time is the transfer speed between the 2 machines, reading directly from the <code class="docutils literal notranslate"><span class="pre">gpfs</span></code>  directory circumvents this therefore.</p>
 </section>
 <section id="additional-considerations">
-<h2>Additional Considerations<a class="headerlink" href="#additional-considerations" title="Link to this heading"></a></h2>
+<h2>Additional considerations<a class="headerlink" href="#additional-considerations" title="Link to this heading"></a></h2>
 <p>When checking if an experiment is stored in a location with the corresponding domain, resolution, and species, consider that the species might not always be listed under the same name.</p>
 <p>The file <code class="docutils literal notranslate"><span class="pre">internal/mapping_species.yaml</span></code> contains a dictionary mapping original species names to their alternative names.</p>
 <p>Note that the mapping species file is only used when the species name from the configuration file is not found in the expected location, meaning Povidentia first looks for the species written in the configuration file. If it is not found, it then searches for the corresponding mapped species in <code class="docutils literal notranslate"><span class="pre">mapping_species.yaml</span></code>.</p>
 </section>
+<section id="different-resolutions-between-model-and-observations">
+<h2>Different resolutions between model and observations<a class="headerlink" href="#different-resolutions-between-model-and-observations" title="Link to this heading"></a></h2>
+<p>When the resolutions are different the model data downsamples to the observations resolution. For instance, if we have 3 hourly model data and daily observations, the model data will be resampled to daily. For this to happen, we need to set the resolution in our configuration file as the obervations frequency.</p>
+</section>
 </section>
 
 

docs/build/doctrees/Interpolation.doctree

@@ -60,7 +60,7 @@
 <li class="toctree-l2"><a class="reference internal" href="#check-that-all-reports-are-created">Check that all reports are created</a></li>
 <li class="toctree-l2"><a class="reference internal" href="#run-providentia-inside-docker-container">Run Providentia inside Docker container</a></li>
 <li class="toctree-l2"><a class="reference internal" href="#generate-the-docs">Generate the docs</a></li>
-<li class="toctree-l2"><a class="reference internal" href="#create-providentia-env-v2-4-0-in-mn5">Create providentia-env_v2.4.0 in MN5</a></li>
+<li class="toctree-l2"><a class="reference internal" href="#create-providentia-env-v3-0-0-in-mn5">Create providentia-env_v3.0.0 in MN5</a></li>
 <li class="toctree-l2"><a class="reference internal" href="#create-local-environment">Create local environment</a></li>
 <li class="toctree-l2"><a class="reference internal" href="#memory-profiling-code">Memory Profiling Code</a><ul>
 <li class="toctree-l3"><a class="reference internal" href="#setup">Setup</a></li>
@@ -106,7 +106,7 @@ <h1>Tips and tricks for developers<a class="headerlink" href="#tips-and-tricks-f
 <li><p><a class="reference internal" href="#check-that-all-reports-are-created">Check that all reports are created</a></p></li>
 <li><p><a class="reference internal" href="#run-providentia-inside-docker-container">Run Providentia inside Docker container</a></p></li>
 <li><p><a class="reference internal" href="#generate-the-docs">Generate the docs</a></p></li>
-<li><p><a class="reference internal" href="#create-providentia-env-v2-4-0-in-mn5">Create providentia-env_v2.4.0 in MN5</a></p></li>
+<li><p><a class="reference internal" href="#create-providentia-env-v3-0-0-in-mn5">Create providentia-env_v3.0.0 in MN5</a></p></li>
 <li><p><a class="reference internal" href="#create-local-environment">Create local environment</a></p></li>
 <li><p><a class="reference internal" href="#memory-profiling-code">Memory Profiling Code</a></p></li>
 </ul>
@@ -269,7 +269,7 @@ <h2>Migration<a class="headerlink" href="#migration" title="Link to this heading
 <section id="run-tests">
 <h2>Run tests<a class="headerlink" href="#run-tests" title="Link to this heading"></a></h2>
 <p>To run all the pipeline tests in your local machine, just run:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">activate</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env</span>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">activate</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env_v3</span><span class="mf">.0.0</span>
 <span class="n">pytest</span> <span class="n">tests</span>
 </pre></div>
 </div>
@@ -358,15 +358,17 @@ <h2>Run Providentia inside Docker container<a class="headerlink" href="#run-prov
 <section id="generate-the-docs">
 <h2>Generate the docs<a class="headerlink" href="#generate-the-docs" title="Link to this heading"></a></h2>
 <p>First edit the .md files under docs/source, then navigate to docs and simply run:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">make</span> <span class="n">clean</span>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">cd</span> <span class="n">docs</span>
+<span class="n">conda</span> <span class="n">activate</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env_v3</span><span class="mf">.0.0</span>
+<span class="n">make</span> <span class="n">clean</span>
 <span class="n">make</span> <span class="n">html</span>
 </pre></div>
 </div>
 </section>
-<section id="create-providentia-env-v2-4-0-in-mn5">
-<h2>Create providentia-env_v2.4.0 in MN5<a class="headerlink" href="#create-providentia-env-v2-4-0-in-mn5" title="Link to this heading"></a></h2>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">create</span> <span class="o">-</span><span class="n">p</span> <span class="o">/</span><span class="n">gpfs</span><span class="o">/</span><span class="n">projects</span><span class="o">/</span><span class="n">bsc32</span><span class="o">/</span><span class="n">repository</span><span class="o">/</span><span class="n">apps</span><span class="o">/</span><span class="n">conda_envs</span><span class="o">/</span><span class="n">providentia</span><span class="o">-</span><span class="n">env_v2</span><span class="mf">.4.0</span> <span class="o">-</span><span class="n">y</span> <span class="n">python</span><span class="o">=</span><span class="mf">3.11.5</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
-<span class="n">conda</span> <span class="n">activate</span> <span class="o">/</span><span class="n">gpfs</span><span class="o">/</span><span class="n">projects</span><span class="o">/</span><span class="n">bsc32</span><span class="o">/</span><span class="n">repository</span><span class="o">/</span><span class="n">apps</span><span class="o">/</span><span class="n">conda_envs</span><span class="o">/</span><span class="n">providentia</span><span class="o">-</span><span class="n">env_v2</span><span class="mf">.4.0</span>
+<section id="create-providentia-env-v3-0-0-in-mn5">
+<h2>Create providentia-env_v3.0.0 in MN5<a class="headerlink" href="#create-providentia-env-v3-0-0-in-mn5" title="Link to this heading"></a></h2>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">create</span> <span class="o">-</span><span class="n">p</span> <span class="o">/</span><span class="n">gpfs</span><span class="o">/</span><span class="n">projects</span><span class="o">/</span><span class="n">bsc32</span><span class="o">/</span><span class="n">repository</span><span class="o">/</span><span class="n">apps</span><span class="o">/</span><span class="n">conda_envs</span><span class="o">/</span><span class="n">providentia</span><span class="o">-</span><span class="n">env_v3</span><span class="mf">.0.0</span> <span class="o">-</span><span class="n">y</span> <span class="n">python</span><span class="o">=</span><span class="mf">3.11.5</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
+<span class="n">conda</span> <span class="n">activate</span> <span class="o">/</span><span class="n">gpfs</span><span class="o">/</span><span class="n">projects</span><span class="o">/</span><span class="n">bsc32</span><span class="o">/</span><span class="n">repository</span><span class="o">/</span><span class="n">apps</span><span class="o">/</span><span class="n">conda_envs</span><span class="o">/</span><span class="n">providentia</span><span class="o">-</span><span class="n">env_v3</span><span class="mf">.0.0</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">cartopy</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">jupyterlab</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">ghostscript</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
@@ -377,16 +379,16 @@ <h2>Create providentia-env_v2.4.0 in MN5<a class="headerlink" href="#create-prov
 <section id="create-local-environment">
 <h2>Create local environment<a class="headerlink" href="#create-local-environment" title="Link to this heading"></a></h2>
 <p>In order to test modules with pip, you need to create an environment. Once activated, you can start installing modules using either <code class="docutils literal notranslate"><span class="pre">pip</span></code> or <code class="docutils literal notranslate"><span class="pre">conda</span></code>, as in this example:</p>
-<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">create</span> <span class="o">-</span><span class="n">n</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env</span> <span class="n">python</span><span class="o">=</span><span class="mf">3.11.5</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
-<span class="n">conda</span> <span class="n">activate</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env</span>
+<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="n">conda</span> <span class="n">create</span> <span class="o">-</span><span class="n">n</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env_v3</span><span class="mf">.0.0</span> <span class="n">python</span><span class="o">=</span><span class="mf">3.11.5</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
+<span class="n">conda</span> <span class="n">activate</span> <span class="n">providentia</span><span class="o">-</span><span class="n">env_v3</span><span class="mf">.0.0</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">cartopy</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">jupyterlab</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">ghostscript</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
 <span class="n">conda</span> <span class="n">install</span> <span class="o">-</span><span class="n">c</span> <span class="n">conda</span><span class="o">-</span><span class="n">forge</span> <span class="n">dask</span> <span class="o">--</span><span class="n">override</span><span class="o">-</span><span class="n">channels</span>
 <span class="n">pip</span> <span class="n">install</span> <span class="o">-</span><span class="n">r</span> <span class="n">requirements</span><span class="o">.</span><span class="n">txt</span>
 </pre></div>
 </div>
-<p>Here we create an environment called <code class="docutils literal notranslate"><span class="pre">providentia-env</span></code> with the Python version 3.11.5, and we install the latest version of Cartopy (with conda, with pip it gives problems), and the Python packages from <code class="docutils literal notranslate"><span class="pre">requirements.txt</span></code> using pip.</p>
+<p>Here we create an environment called <code class="docutils literal notranslate"><span class="pre">providentia-env_v3.00</span></code> with the Python version 3.11.5, and we install the latest version of Cartopy (with conda, with pip it gives problems), and the Python packages from <code class="docutils literal notranslate"><span class="pre">requirements.txt</span></code> using pip.</p>
 <p>pip install memray</p>
 <p>memray run -o result.bin python your_script.py
 memray flamegraph result.bin  # Creates a flamegraph