Merge branch 'check_fof_update' of https://github.com/MeteoSwiss/probtest into check_fof_update

Author: cghielmini

engine/check.py

@@ -43,13 +43,8 @@
     default="",
     help=cli_help["fof_types"],
 )
-def check(
-    reference_files,
-    current_files,
-    tolerance_files,
-    factor,
-    fof_types,
-):
+@click.option("--rules", default="")
+def check(reference_files, current_files, tolerance_files, factor, fof_types, rules):
 
     zipped = zip(reference_files, current_files, tolerance_files)
 
@@ -60,7 +55,7 @@ def check(
     for reference_file, current_file, tolerance_file in expanded_zip:
 
         out, err, tol = check_file_with_tolerances(
-            tolerance_file, reference_file, current_file, factor
+            tolerance_file, reference_file, current_file, factor, rules
         )
 
         if out:

tests/cli/test_tolerance_cli_split.py

@@ -0,0 +1,90 @@
+"""
+This module contains tests for verifying the correctness of the tolerance CLI
+command.
+The tests compare the output of the tolerance CLI with reference tolerance data
+and check for discrepancies.
+"""
+
+import os
+
+import pandas as pd
+import pytest
+
+from tests.helpers import (
+    assert_empty_df,
+    load_pandas,
+    pandas_error,
+    run_tolerance_cli,
+    store_as_potential_new_ref,
+)
+
+
+@pytest.mark.parametrize("use_minimum_tolerance", [True, False])
+def test_tolerance_cli_stats(
+    ref_data, df_ref_tolerance, tmp_dir, new_ref, use_minimum_tolerance
+):
+
+    stats_file_name = os.path.join(ref_data, "stats_{member_type}{member_id}.csv")
+    tolerance_file_name = os.path.join(tmp_dir, "tolerance.csv")
+
+    if use_minimum_tolerance:
+        run_tolerance_cli(
+            stats_file_name,
+            tolerance_file_name,
+            member_type="dp_",
+            minimum_tolerance=1e-14,
+        )
+
+        df_test = load_pandas(tolerance_file_name, index_col=[0, 1])
+        err = pandas_error(df_ref_tolerance, df_test)
+
+    else:
+        run_tolerance_cli(
+            stats_file_name,
+            tolerance_file_name,
+            member_type="dp_",
+        )
+
+        df_test = load_pandas(tolerance_file_name, index_col=[0, 1])
+        modified_df_ref_tolerance = df_ref_tolerance
+        modified_df_ref_tolerance.loc[("NetCDF:*.nc", "T")] = 0.0
+
+        err = pandas_error(modified_df_ref_tolerance, df_test)
+
+    store_as_potential_new_ref(tolerance_file_name, new_ref)
+
+    assert_empty_df(err, "Tolerance datasets are not equal!")
+
+
+@pytest.mark.parametrize("use_minimum_tolerance", [True, False])
+def test_tolerance_cli_fof(
+    fof_file_set, df_ref_tolerance_fof, new_ref, use_minimum_tolerance
+):
+
+    fof_pattern = fof_file_set["path"]
+    tolerance_files = fof_file_set["tol"]
+    member_ids = "1,2,3,4"
+
+    for fof_file in fof_file_set["fof"]:
+        assert os.path.exists(fof_file), f"File FOF missing: {fof_file}"
+
+    if use_minimum_tolerance:
+
+        run_tolerance_cli(
+            fof_pattern, tolerance_files, member_ids=member_ids, minimum_tolerance=1e-14
+        )
+
+        df_test = pd.read_csv(tolerance_files[0], index_col=[0])
+        df_test.columns = [None]
+        err = pandas_error(df_ref_tolerance_fof, df_test)
+
+    else:
+        run_tolerance_cli(fof_pattern, tolerance_files, member_ids=member_ids)
+
+        df_test = pd.read_csv(tolerance_files[0], index_col=[0])
+        df_test.columns = [None]
+        err = pandas_error(df_ref_tolerance_fof, df_test)
+
+    store_as_potential_new_ref(tolerance_files[0], new_ref)
+
+    assert_empty_df(err, "Tolerance datasets are not equal!")

tests/engine/test_check.py

@@ -15,7 +15,7 @@
 )
 
 
-@pytest.fixture(name="dataframes", scope="function")
+@pytest.fixture(name="stats_dataframes", scope="function")
 def fixture_dataframes():
     """
     Create stats dataframes and reference tolerances.
@@ -89,8 +89,8 @@ def _check(df1, df2, tol_large, tol_small, file_type):
     )
 
 
-def test_check(dataframes):
-    df1, df2, tol_large, tol_small = dataframes
+def test_check_stats(stats_dataframes):
+    df1, df2, tol_large, tol_small = stats_dataframes
     _check(df1, df2, tol_large, tol_small, file_type="stats")
 
 
@@ -109,7 +109,7 @@ def test_check_fof(fof_datasets):
     )
 
 
-def test_check_one_zero(dataframes):
+def test_check_one_zero_stats(dataframes):
     """
     Test that a null value in ds1 causes failure,
     and that a variation within tolerance is accepted.
@@ -122,7 +122,7 @@ def test_check_one_zero(dataframes):
     diff_df = diff_df.groupby(["variable"]).max()
     out, err, _ = check_variable(diff_df, tol_large)
 
-    assert not out, f"Check con 0-value reference ha validato erroneamente:\n{err}"
+    assert not out, f"Check with 0-value reference validated incorrectly:\n{err}"
 
     df2 = df2.copy()
     df2.loc[("NetCDF:*atm_3d*.nc", "var_1", 2), (0, "max")] = CHECK_THRESHOLD / 2
@@ -149,7 +149,7 @@ def test_check_one_zero_fof(fof_datasets):
 
     out, err, _ = check_variable(diff_df, tol_large)
 
-    assert not out, f"Check con 0-value reference ha validato erroneamente:\n{err}"
+    assert not out, f"Check with 0-value reference validated incorrectly:\n{err}"
 
     _, _, ds_veri2_copy = split_feedback_dataset(ds2_copy)
     ds_veri2_copy = ds_veri2_copy.copy(deep=True)
@@ -163,7 +163,7 @@ def test_check_one_zero_fof(fof_datasets):
     )
 
 
-def test_check_smalls(dataframes):
+def test_check_smalls_stats(dataframes):
     """
     Both values are close to 0 and should be accepted even though
     their relative difference is large.

tests/engine/test_check2.py

@@ -13,7 +13,7 @@
 from engine.check import check
 
 
-@pytest.fixture(name="dataframes", scope="function")
+@pytest.fixture(name="stats_dataframes", scope="function")
 def fixture_dataframes(tmp_dir):
     """
     Create stats dataframes and reference tolerances.
@@ -94,13 +94,13 @@ def fixture_fof_datasets(tmp_dir, sample_dataset_fof):
     yield ds1_file, ds2_file, tol_large_file, tol_small_file
 
 
-def test_check_cli_stats(dataframes):
+def test_check_cli_stats(stats_dataframes):
     """
     Test that is not validated in the case of large tolerances but
     is validated in the case of small tolerances.
     """
 
-    df1_stats, df2_stats, tol_large, tol_small = dataframes
+    df1_stats, df2_stats, tol_large, tol_small = stats_dataframes
 
     runner = CliRunner()
     result = runner.invoke(