Merge pull request #49 from xcube-dev/tejas-48-fix-issues

Fix xcube-cmems issues

Author: TejasMorbagal

CHANGES.md

@@ -1,5 +1,11 @@
-## Changes in 0.1.7
+## Changes in 0.1.8
+
+- Made metadata attributes of datasets returned by the data store JSON-serializable to 
+  avoid problems in xcube Server. (#48)
 
+- Disabled progress bar in list_data_ids() to avoid stdout bloats in JNB (#47)
+
+## Changes in 0.1.7
 
 - Ensuring compatibility with latest `copernicusmarine` requirements
 

README.md

@@ -74,10 +74,9 @@ $ pytest
 ```
 
 in the `xcube-cmems` repository. Note that, in order to successfully run the
-tests using the current repository version of `xcube-cmems`, you may also need to
-install the repository source version of `xcube` rather than its latest conda-forge
-release and have the cmems credentials set as environment variables `CMEMS_USERNAME` 
-and `CMEMS_PASSWORD`.
+tests using the current repository version of `xcube-cmems`, you may also need to have 
+the cmems credentials set as environment variables 
+`COPERNICUSMARINE_SERVICE_USERNAME` and `COPERNICUSMARINE_SERVICE_PASSWORD`.
 
 To create a test coverage report, you can use
 
@@ -95,4 +94,4 @@ in the `examples/notebooks/` subdirectory of the repository.
 ## Releasing
 
 To release `xcube-cmems`, please follow the steps outlined in the 
-[xcube Developer Guide](https://github.com/dcs4cop/xcube/blob/master/docs/source/devguide.md#release-process).
+[xcube Developer Guide](https://github.com/dcs4cop/xcube/blob/master/docs/source/devguide.md#release-process).

environment.yml

@@ -6,8 +6,8 @@ dependencies:
   - python >=3.10
   # Required
   - copernicusmarine >=2.1.1
-  - xcube >=1.9.1
-  - numpy >=2.0.0 # to avoid inconsistent results with copernicusmarine package
+  - xcube >=1.11.1
+  - numpy
   - pandas
   # for testing
   - black

examples/notebooks/cmems_example_JNB.ipynb

@@ -22,7 +22,7 @@ requires-python = ">=3.10"
 dependencies = [
     # Todo: add xcube-core when issue with matplotlib-base is solved
     "copernicusmarine>=2.1.1",
-    "numpy>=2.0.0",
+    "numpy",
     "pandas",
     "zarr"
 ]

pyproject.toml

@@ -24,33 +24,35 @@
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 
+import numpy as np
+import xarray as xr
+
 from xcube_cmems.cmems import Cmems
 
 
 class CmemsTest(unittest.TestCase):
-
     def setUp(self):
         # Setup environment variables for testing
-        os.environ["COPERNICUS_MARINE_SERVICE_USERNAME"] = "testuser"
-        os.environ["COPERNICUS_MARINE_SERVICE_PASSWORD"] = "testpass"
+        os.environ["COPERNICUSMARINE_SERVICE_USERNAME"] = "testuser"
+        os.environ["COPERNICUSMARINE_SERVICE_PASSWORD"] = "testpass"
+        self.cmems = Cmems()
 
     @patch("xcube_cmems.cmems.cm.describe")
     def test_get_datasets_with_titles(self, mock_describe):
-        # Fake datasets
+        # Mock datasets
         dataset1 = SimpleNamespace(dataset_id="dataset1", dataset_name="Dataset 1")
         dataset2 = SimpleNamespace(dataset_id="dataset2", dataset_name="Dataset 2")
         dataset3 = SimpleNamespace(dataset_id="dataset3", dataset_name="Dataset 3")
 
-        # Fake products
+        # Mock products
         product_a = SimpleNamespace(title="Product A", datasets=[dataset1, dataset2])
         product_b = SimpleNamespace(title="Product B", datasets=[dataset3])
 
-        # Fake catalogue
+        # Mock catalogue
         mock_catalogue = SimpleNamespace(products=[product_a, product_b])
         mock_describe.return_value = mock_catalogue
 
-        cmems = Cmems()
-        datasets_info = cmems.get_datasets_with_titles()
+        datasets_info = self.cmems.get_datasets_with_titles()
 
         expected = [
             {"dataset_id": "dataset1", "title": "Product A - Dataset 1"},
@@ -64,19 +66,58 @@ def test_open_dataset(self, mock_open_dataset):
         # Mock the response from cm.open_dataset
         mock_dataset = MagicMock()
         mock_open_dataset.return_value = mock_dataset
-        cmems_instance = Cmems()
-        result = cmems_instance.open_dataset("dataset1")
+        result = self.cmems.open_dataset("dataset1")
         self.assertEqual(result, mock_dataset)
 
         # Testing with a non-existing dataset
         mock_open_dataset.side_effect = KeyError("Dataset not found")
-        result = cmems_instance.open_dataset("non_existing_dataset")
+        result = self.cmems.open_dataset("non_existing_dataset")
         self.assertIsNone(result)
 
     @patch("click.confirm", return_value=True)
     def test_open_data_for_not_exsiting_dataset(self, mock_confirm):
-        cmems = Cmems()
         self.assertIsNone(
-            cmems.open_dataset("dataset-bal-analysis-forecast" "-wav-hourly"),
+            self.cmems.open_dataset("dataset-bal-analysis-forecast" "-wav-hourly"),
             "Expected the method to return None for a " "non-existing dataset",
         )
+
+    def test_to_json_serializable_scalar_types(self):
+        assert self.cmems.to_json_serializable(np.int16(5)) == 5
+        assert self.cmems.to_json_serializable(np.bool_(True)) is True
+
+    def test_to_json_serializable_ndarray(self):
+        arr = np.array([1, 2, 3], dtype=np.int32)
+        result = self.cmems.to_json_serializable(arr)
+        assert result == [1, 2, 3]
+        assert isinstance(result, list)
+
+    def test_to_json_serializable_nested_structures(self):
+        obj = {
+            "a": np.int32(1),
+            "b": [np.float64(2.2), {"c": np.bool_(False)}],
+            "d": (np.int8(4),),
+        }
+        expected = {"a": 1, "b": [2.2, {"c": False}], "d": [4]}
+        assert self.cmems.to_json_serializable(obj) == expected
+
+    def test_sanitize_attrs_dataset(self):
+        data = xr.Dataset(
+            {
+                "var1": xr.DataArray(
+                    np.random.rand(2, 2),
+                    dims=["x", "y"],
+                    attrs={"int_attr": np.int16(10), "float_attr": np.float64(2.5)},
+                )
+            },
+            attrs={"global_attr": np.bool_(True)},
+        )
+
+        sanitized = self.cmems.sanitize_attrs(data)
+
+        # Ensure all attributes are native Python types
+        for k, v in sanitized.attrs.items():
+            assert isinstance(v, (int, float, bool, str, list, dict))
+
+        for var in sanitized.data_vars:
+            for k, v in sanitized[var].attrs.items():
+                assert isinstance(v, (int, float, bool, str, list, dict))

test/test_cmems.py

@@ -33,7 +33,6 @@
 
 
 class CmemsDataOpenerTest(unittest.TestCase):
-
     def setUp(self) -> None:
         self.dataset_id = "cmems_mod_arc_bgc_anfc_ecosmo_P1D-m"
         self.opener = CmemsDatasetOpener()
@@ -100,7 +99,6 @@ def test_describe_data(self, mock_open_dataset):
 
 
 class CmemsDataStoreTest(unittest.TestCase):
-
     def setUp(self) -> None:
         self.dataset_id = "cmems_mod_arc_bgc_anfc_ecosmo_P1D-m"
         self.mock_datasets = [
@@ -191,17 +189,16 @@ def test_get_data_opener_ids_with_valid_data_id(self):
 
 
 class CmemsDataStoreParamsTest(unittest.TestCase):
-
     def test_store_for_cmems_credentials(self):
         params = {"cmems_username": "", "cmems_password": ""}
         with self.assertRaises(Exception) as e:
             CmemsDataStore(**params)
         self.assertEqual(
-            "CmemsDataStore needs cmems credentials to "
-            "be provided either as "
-            "environment variables CMEMS_USERNAME and "
-            "CMEMS_PASSWORD, or to be "
-            "provided as store params cmems_username and "
-            "cmems_password",
+             "CmemsDataStore needs cmems credentials to "
+                "be provided either as "
+                "environment variables COPERNICUSMARINE_SERVICE_USERNAME and "
+                "COPERNICUSMARINE_SERVICE_PASSWORD, or to be "
+                "provided as store params cmems_username and "
+                "cmems_password",
             f"{e.exception}",
         )

test/test_store.py

@@ -23,40 +23,39 @@
 from typing import List, Optional
 
 import copernicusmarine as cm
+import numpy as np
 import xarray as xr
+from copernicusmarine import CopernicusMarineCatalogue
 
 
 class Cmems:
-
     def __init__(
-        self,
-        cmems_username: Optional[str] = None,
-        cmems_password: Optional[str] = None,
+        self, cmems_username: Optional[str] = None, cmems_password: Optional[str] = None
     ):
         self.cmems_username = (
             cmems_username
             if cmems_username is not None
-            else os.getenv("CMEMS_USERNAME")
+            else os.getenv("COPERNICUSMARINE_SERVICE_USERNAME")
         )
         self.cmems_password = (
             cmems_password
             if cmems_password is not None
-            else os.getenv("CMEMS_PASSWORD")
+            else os.getenv("COPERNICUSMARINE_SERVICE_PASSWORD")
         )
 
         if not self.cmems_username or not self.cmems_password:
             raise ValueError(
                 "CmemsDataStore needs cmems credentials to "
                 "be provided either as "
-                "environment variables CMEMS_USERNAME and "
-                "CMEMS_PASSWORD, or to be "
+                "environment variables COPERNICUSMARINE_SERVICE_USERNAME and "
+                "COPERNICUSMARINE_SERVICE_PASSWORD, or to be "
                 "provided as store params cmems_username and "
                 "cmems_password"
             )
 
     @classmethod
     def get_datasets_with_titles(cls) -> List[dict]:
-        catalogue: CopernicusMarineCatalogue = cm.describe()
+        catalogue: CopernicusMarineCatalogue = cm.describe(disable_progress_bar=True)
         datasets_info: List[dict] = []
         for product in catalogue.products:
             product_title = product.title
@@ -69,6 +68,31 @@ def get_datasets_with_titles(cls) -> List[dict]:
                 )
         return datasets_info
 
+    def to_json_serializable(self, obj):
+        """Convert NumPy types and nested structures to JSON-serializable types."""
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.bool_):
+            return bool(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        elif isinstance(obj, dict):
+            return {str(k): self.to_json_serializable(v) for k, v in obj.items()}
+        elif isinstance(obj, (list, tuple)):
+            return [self.to_json_serializable(i) for i in obj]
+        return obj
+
+    def sanitize_attrs(self, ds: xr.Dataset) -> xr.Dataset:
+        """Sanitize dataset and variable attributes for JSON serialization."""
+        for var in ds.data_vars:
+            ds[var].attrs = {
+                str(k): self.to_json_serializable(v) for k, v in ds[var].attrs.items()
+            }
+        ds.attrs = {str(k): self.to_json_serializable(v) for k, v in ds.attrs.items()}
+        return ds
+
     def open_dataset(self, dataset_id, **open_params) -> xr.Dataset:
         try:
 
@@ -78,6 +102,7 @@ def open_dataset(self, dataset_id, **open_params) -> xr.Dataset:
                 password=self.cmems_password,
                 **open_params,
             )
+            ds = self.sanitize_attrs(ds)
             return ds
         except KeyError as e:
             print(f"Error: {e}.")

xcube_cmems/cmems.py

@@ -176,13 +176,8 @@ def get_open_data_params_schema(self, data_id: str = None) -> JsonObjectSchema:
 
 
 class CmemsDatasetOpener(CmemsDataOpener):
-
     def __init__(self, **cmems_params):
-        super().__init__(
-            Cmems(**cmems_params),
-            DATASET_OPENER_ID,
-            DATASET_TYPE,
-        )
+        super().__init__(Cmems(**cmems_params), DATASET_OPENER_ID, DATASET_TYPE)
 
 
 class CmemsDataStore(DataStore):

xcube_cmems/store.py

@@ -19,4 +19,4 @@
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
-version = "0.1.7"
+version = "0.1.8.dev0"