remove secrets from config while parsing private config (ecmwf#269)

* remove secrets from dataloader while parsing private config

Author: grassesi

src/weathergen/utils/config.py

@@ -38,8 +38,6 @@ def _format_cf(config: Config) -> str:
     stream = io.StringIO()
     for key, value in config.items():
         match key:
-            case "secrets":
-                continue
             case "streams":
                 for rt in value:
                     for k, v in rt.items():
@@ -103,7 +101,8 @@ def load_config(
     *overwrites: Path | dict | Config,
 ) -> Config:
     """
-    Merge config information from multiple sources into one run_config.
+    Merge config information from multiple sources into one run_config. Anything in the
+    private configs "secrets" section will be discarted.
 
     Args:
         private_home: Configuration file containing platform dependent information and secretes
@@ -211,6 +210,7 @@ def _load_private_conf(private_home: Path | None) -> DictConfig:
     private_cf["model_path"] = (
         private_cf["model_path"] if "model_path" in private_cf.keys() else "./models"
     )
+    del private_cf["secrets"]
     return private_cf
 
 

tests/test_config.py

@@ -150,7 +150,9 @@ def config_fresh(private_config_file):
 
 
 def test_contains_private(config_fresh):
-    assert contains_keys(config_fresh, DUMMY_PRIVATE_CONF)
+    sanitized_private_conf = DUMMY_PRIVATE_CONF.copy()
+    del sanitized_private_conf["secrets"]
+    assert contains_keys(config_fresh, sanitized_private_conf)
 
 
 @pytest.mark.parametrize("overwrite_dict", DUMMY_OVERWRITES, indirect=True)
@@ -207,9 +209,8 @@ def test_from_cli(options, cf):
 
 def test_print_cf_no_secrets(config_fresh):
     output = config._format_cf(config_fresh)
-    print(output)
 
-    assert "53CR3T" not in output
+    assert "53CR3T" not in output and "secrets" not in config_fresh.keys()
 
 
 @pytest.mark.parametrize("streams_dir", VALID_STREAMS, indirect=True)