From 3bff90dc1163f24a9344d7a2fb50530a3d7fa459 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 19 May 2025 10:20:36 +0200 Subject: [PATCH 01/15] use new dtypes branch of zarr python --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fd4a4293882..2ccfe48b2ac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ io = [ "h5netcdf", "scipy", 'pydap; python_version<"3.10"', - "zarr", + "zarr @ git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings", "fsspec", "cftime", "pooch", From 3dc64fc90abe9735686a55ce4b12f1bd3bf6b84e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 10:19:23 +0200 Subject: [PATCH 02/15] special-case for zarr in module_available --- xarray/namedarray/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 96060730345..a3a2977a644 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -53,7 +53,14 @@ def module_available(module: str, minversion: str | None = None) -> bool: return False if minversion is not None: - version = importlib.metadata.version(module) + # special case for zarr, because importlib.metadata.version doesn't retrieve the correct + # zarr version under some circumstances. See https://github.com/pydata/xarray/issues/10335 + if module == "zarr": + import zarr + + version = zarr.__version__ + else: + version = importlib.metadata.version(module) return Version(version) >= Version(minversion) From 3c40a045061a59d90a3b95020316203671b55bde Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 17:38:05 +0200 Subject: [PATCH 03/15] try declaring git dep in environment.yml --- ci/requirements/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index b4354b14f40..e7d5fc264fd 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -58,8 +58,8 @@ dependencies: - types-setuptools - types-openpyxl - typing_extensions - - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present - types-defusedxml - types-pexpect + - git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings=zarr From a7826fe8fab6e0504752a64941210ca7e11c8a4d Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 18:10:13 +0200 Subject: [PATCH 04/15] try another dep site --- ci/install-upstream-wheels.sh | 2 +- ci/requirements/environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 02d36259f82..ce3f5996680 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -53,7 +53,7 @@ python -m pip install \ git+https://github.com/dask/dask \ git+https://github.com/dask/dask-expr \ git+https://github.com/dask/distributed \ - git+https://github.com/zarr-developers/zarr-python \ + git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings \ git+https://github.com/Unidata/cftime \ git+https://github.com/pypa/packaging \ git+https://github.com/hgrecco/pint \ diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index e7d5fc264fd..b4354b14f40 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -58,8 +58,8 @@ dependencies: - types-setuptools - types-openpyxl - typing_extensions + - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present - types-defusedxml - types-pexpect - - git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings=zarr From 5aa685bd40d08e68cb37e568f17532ab873131eb Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 19:08:11 +0200 Subject: [PATCH 05/15] fix zarr version detection --- xarray/backends/zarr.py | 8 +++++++- xarray/namedarray/utils.py | 10 +--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 1a46346dda7..7924312157e 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -105,7 +105,13 @@ def _choose_default_mode( def _zarr_v3() -> bool: - return module_available("zarr", minversion="3") + # don't use the module_available function because it doesn't report zarr v3 correctly. + try: + from packaging.version import Version + import zarr + return Version(zarr.__version__).major == 3 + except ImportError: + return False # need some special secret attributes to tell us the dimensions diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index a3a2977a644..9e1e6e7cb7d 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -51,16 +51,8 @@ def module_available(module: str, minversion: str | None = None) -> bool: """ if importlib.util.find_spec(module) is None: return False - if minversion is not None: - # special case for zarr, because importlib.metadata.version doesn't retrieve the correct - # zarr version under some circumstances. See https://github.com/pydata/xarray/issues/10335 - if module == "zarr": - import zarr - - version = zarr.__version__ - else: - version = importlib.metadata.version(module) + version = importlib.metadata.version(module) return Version(version) >= Version(minversion) From 5aca9c11f08b07516489b14b9b95b893bb3167ab Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 May 2025 17:09:00 +0000 Subject: [PATCH 06/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/backends/zarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 7924312157e..6e127a8f4a6 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -35,7 +35,6 @@ from xarray.core.variable import Variable from xarray.namedarray.parallelcompat import guess_chunkmanager from xarray.namedarray.pycompat import integer_types -from xarray.namedarray.utils import module_available if TYPE_CHECKING: from xarray.backends.common import AbstractDataStore @@ -107,8 +106,9 @@ def _choose_default_mode( def _zarr_v3() -> bool: # don't use the module_available function because it doesn't report zarr v3 correctly. try: - from packaging.version import Version import zarr + from packaging.version import Version + return Version(zarr.__version__).major == 3 except ImportError: return False From 34630156ff3f62210ee86678092b99ac9cd4493e Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Fri, 23 May 2025 14:49:25 +0200 Subject: [PATCH 07/15] use numpy dtype exposed by zarr array instead of metadata.data_type --- xarray/backends/zarr.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 1a46346dda7..bb502d52da0 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -877,9 +877,8 @@ def open_store_variable(self, name): if zarr_array.fill_value is not None: attributes["_FillValue"] = zarr_array.fill_value elif "_FillValue" in attributes: - original_zarr_dtype = zarr_array.metadata.data_type attributes["_FillValue"] = FillValueCoder.decode( - attributes["_FillValue"], original_zarr_dtype.value + attributes["_FillValue"], zarr_array.dtype ) return Variable(dimensions, data, attributes, encoding) From 3a264d107978ffe347fb0c953927d44f847c8458 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Mon, 19 May 2025 10:20:36 +0200 Subject: [PATCH 08/15] use new dtypes branch of zarr python --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 63d1231b11c..a8fd694a9e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ io = [ "h5netcdf", "scipy", 'pydap; python_version<"3.10"', - "zarr", + "zarr @ git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings", "fsspec", "cftime", "pooch", From 9808a796919802f184adc2ac251e63224f1c371c Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 10:19:23 +0200 Subject: [PATCH 09/15] special-case for zarr in module_available --- xarray/namedarray/utils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index 96060730345..a3a2977a644 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -53,7 +53,14 @@ def module_available(module: str, minversion: str | None = None) -> bool: return False if minversion is not None: - version = importlib.metadata.version(module) + # special case for zarr, because importlib.metadata.version doesn't retrieve the correct + # zarr version under some circumstances. See https://github.com/pydata/xarray/issues/10335 + if module == "zarr": + import zarr + + version = zarr.__version__ + else: + version = importlib.metadata.version(module) return Version(version) >= Version(minversion) From 356811b87d752bbb676c1f4e6ede7fe018095dba Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 17:38:05 +0200 Subject: [PATCH 10/15] try declaring git dep in environment.yml --- ci/requirements/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index b4354b14f40..e7d5fc264fd 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -58,8 +58,8 @@ dependencies: - types-setuptools - types-openpyxl - typing_extensions - - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present - types-defusedxml - types-pexpect + - git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings=zarr From 19f463602d8794348bab235fc993497b6c16365b Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 18:10:13 +0200 Subject: [PATCH 11/15] try another dep site --- ci/install-upstream-wheels.sh | 2 +- ci/requirements/environment.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index 02d36259f82..ce3f5996680 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -53,7 +53,7 @@ python -m pip install \ git+https://github.com/dask/dask \ git+https://github.com/dask/dask-expr \ git+https://github.com/dask/distributed \ - git+https://github.com/zarr-developers/zarr-python \ + git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings \ git+https://github.com/Unidata/cftime \ git+https://github.com/pypa/packaging \ git+https://github.com/hgrecco/pint \ diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml index e7d5fc264fd..b4354b14f40 100644 --- a/ci/requirements/environment.yml +++ b/ci/requirements/environment.yml @@ -58,8 +58,8 @@ dependencies: - types-setuptools - types-openpyxl - typing_extensions + - zarr - pip: - jax # no way to get cpu-only jaxlib from conda if gpu is present - types-defusedxml - types-pexpect - - git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings=zarr From 52dfd507fee1cabd8df39303991009e30eab9e95 Mon Sep 17 00:00:00 2001 From: Davis Vann Bennett Date: Tue, 20 May 2025 19:08:11 +0200 Subject: [PATCH 12/15] fix zarr version detection --- xarray/backends/zarr.py | 8 +++++++- xarray/namedarray/utils.py | 10 +--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index bb502d52da0..fce3343aaac 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -105,7 +105,13 @@ def _choose_default_mode( def _zarr_v3() -> bool: - return module_available("zarr", minversion="3") + # don't use the module_available function because it doesn't report zarr v3 correctly. + try: + from packaging.version import Version + import zarr + return Version(zarr.__version__).major == 3 + except ImportError: + return False # need some special secret attributes to tell us the dimensions diff --git a/xarray/namedarray/utils.py b/xarray/namedarray/utils.py index a3a2977a644..9e1e6e7cb7d 100644 --- a/xarray/namedarray/utils.py +++ b/xarray/namedarray/utils.py @@ -51,16 +51,8 @@ def module_available(module: str, minversion: str | None = None) -> bool: """ if importlib.util.find_spec(module) is None: return False - if minversion is not None: - # special case for zarr, because importlib.metadata.version doesn't retrieve the correct - # zarr version under some circumstances. See https://github.com/pydata/xarray/issues/10335 - if module == "zarr": - import zarr - - version = zarr.__version__ - else: - version = importlib.metadata.version(module) + version = importlib.metadata.version(module) return Version(version) >= Version(minversion) From bf1cf51107515dc68cd9a3fa8ca0edf7ce90cb8e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 20 May 2025 17:09:00 +0000 Subject: [PATCH 13/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- xarray/backends/zarr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index fce3343aaac..ed7db59a308 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -35,7 +35,6 @@ from xarray.core.variable import Variable from xarray.namedarray.parallelcompat import guess_chunkmanager from xarray.namedarray.pycompat import integer_types -from xarray.namedarray.utils import module_available if TYPE_CHECKING: from xarray.backends.common import AbstractDataStore @@ -107,8 +106,9 @@ def _choose_default_mode( def _zarr_v3() -> bool: # don't use the module_available function because it doesn't report zarr v3 correctly. try: - from packaging.version import Version import zarr + from packaging.version import Version + return Version(zarr.__version__).major == 3 except ImportError: return False From 601e5ea8fe56572d36dbbcef3266b8dd5097065a Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 3 Jul 2025 22:46:43 +0200 Subject: [PATCH 14/15] Update install-upstream-wheels.sh --- ci/install-upstream-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index ce3f5996680..fa0b37d648d 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -53,7 +53,7 @@ python -m pip install \ git+https://github.com/dask/dask \ git+https://github.com/dask/dask-expr \ git+https://github.com/dask/distributed \ - git+https://github.com/d-v-b/zarr-python.git@feat/fixed-length-strings \ + git+https://github.com/zarr-python/zarr-python.git@main \ git+https://github.com/Unidata/cftime \ git+https://github.com/pypa/packaging \ git+https://github.com/hgrecco/pint \ From ce7a18149a086ea77257e8dc3abc1470236b8325 Mon Sep 17 00:00:00 2001 From: Davis Bennett Date: Thu, 3 Jul 2025 22:59:27 +0200 Subject: [PATCH 15/15] Update install-upstream-wheels.sh --- ci/install-upstream-wheels.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index fa0b37d648d..f5cbaa0729b 100755 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -53,7 +53,7 @@ python -m pip install \ git+https://github.com/dask/dask \ git+https://github.com/dask/dask-expr \ git+https://github.com/dask/distributed \ - git+https://github.com/zarr-python/zarr-python.git@main \ + git+https://github.com/zarr-developers/zarr-python.git@main \ git+https://github.com/Unidata/cftime \ git+https://github.com/pypa/packaging \ git+https://github.com/hgrecco/pint \