Exclude bad version of numcodecs & fix bsddb3 doctests (#2544)

* Exclude bad version of numcodecs

Actually exclude bad version of numcodecs

Fix changelog entry

* Add pytest-doctestplus doc dep

Fix doctest requires

Try bumping version of pytest-doctestplus

Fixup doctest requires

* Exclude other bad version of numcodecs

* Add pytest doctestplus extension

* Fix skipping in docstring

* Remove bsddb3 tests

* Pin max version of numpy

Author: dstansby

docs/conf.py

@@ -44,6 +44,7 @@
     "sphinx_issues",
     "sphinx_copybutton",
     "sphinx_design",
+    "pytest_doctestplus.sphinx.doctestplus",
 ]
 
 numpydoc_show_class_members = False

docs/release.rst

@@ -36,6 +36,9 @@ Maintenance
   ``zarr-python`` if you can install it, but to reduce our maintenance
   burden we will no longer run our compatibility tests for it.
   By :user:`David Stansby <dstansby>` (:issue:`2344`).
+* Excluded versions 0.14.0 and 0.14.1 of numcodecs, due to a bug in the implementation of
+  the Delta filter (see https://github.com/zarr-developers/numcodecs/issues/653 for more information).
+  By :user:`David Stansby <dstansby>` (:issue:`2544`).
 
 Deprecations
 ~~~~~~~~~~~~

docs/tutorial.rst

@@ -809,7 +809,7 @@ Another storage alternative is the :class:`zarr.storage.DBMStore` class, added
 in Zarr version 2.2. This class allows any DBM-style database to be used for
 storing an array or group. Here is an example using a Berkeley DB B-tree
 database for storage (requires `bsddb3
-<https://www.jcea.es/programacion/pybsddb.htm>`_ to be installed)::
+<https://www.jcea.es/programacion/pybsddb.htm>`_ to be installed):
 
 .. doctest-requires:: bsddb3
 

pyproject.toml

@@ -11,9 +11,9 @@ maintainers = [{ name = "Alistair Miles", email = "alimanfoo@googlemail.com" }]
 requires-python = ">=3.11"
 dependencies = [
     'asciitree',
-    'numpy>=1.24',
+    'numpy>=1.24,<2.2',
     'fasteners; sys_platform != "emscripten"',
-    'numcodecs>=0.10.0',
+    'numcodecs>=0.10.0,!=0.14.0,!=0.14.1',
 ]
 dynamic = ["version"]
 classifiers = [
@@ -42,7 +42,8 @@ docs = [
     'sphinx-copybutton',
     'pydata-sphinx-theme',
     'numpydoc',
-    'numcodecs[msgpack]',
+    'numcodecs[msgpack]!=0.14.0,!=0.14.1',
+    'pytest-doctestplus',
 ]
 
 [project.urls]

requirements_dev_optional.txt

@@ -15,7 +15,7 @@ pymongo==4.10.1
 # optional test requirements
 coverage
 pytest-cov==5.0.0
-pytest-doctestplus==1.2.1
+pytest-doctestplus==1.3.0
 pytest-timeout==2.3.1
 h5py==3.12.1
 fsspec==2023.12.2

zarr/storage.py

@@ -2069,13 +2069,11 @@ class DBMStore(Store):
     <https://www.jcea.es/programacion/pybsddb.htm>`_ package is installed, a
     Berkeley DB database can be used::
 
-    .. doctest-requires:: bsddb3
-
-        >>> import bsddb3
-        >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen)
-        >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True)
-        >>> z[...] = 42
-        >>> store.close()
+        >>> import bsddb3  # doctest: +SKIP
+        >>> store = zarr.DBMStore('data/array.bdb', open=bsddb3.btopen) # doctest: +SKIP
+        >>> z = zarr.zeros((10, 10), chunks=(5, 5), store=store, overwrite=True) # doctest: +SKIP
+        >>> z[...] = 42 # doctest: +SKIP
+        >>> store.close() # doctest: +SKIP
 
     Notes
     -----

zarr/tests/test_core.py

@@ -1,5 +1,4 @@
 import atexit
-import os
 import sys
 import pickle
 import shutil
@@ -75,7 +74,6 @@
 from zarr.util import buffer_size
 from zarr.tests.util import (
     abs_container,
-    have_bsddb3,
     have_fsspec,
     have_lmdb,
     have_sqlite3,
@@ -2046,20 +2044,6 @@ def test_nbytes_stored(self):
         pass  # not implemented
 
 
-@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3")
-class TestArrayWithDBMStoreBerkeleyDB(TestArray):
-    def create_store(self):
-        import bsddb3
-
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStore(path, flag="n", open=bsddb3.btopen)
-        return store
-
-    def test_nbytes_stored(self):
-        pass  # not implemented
-
-
 @pytest.mark.skipif(have_lmdb is False, reason="needs lmdb")
 class TestArrayWithLMDBStore(TestArray):
     def create_store(self):
@@ -2767,21 +2751,6 @@ def test_nbytes_stored(self):
         pass  # not implemented
 
 
-@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
-@pytest.mark.skipif(have_bsddb3 is False, reason="needs bsddb3")
-class TestArrayWithDBMStoreV3BerkeleyDB(TestArrayV3):
-    def create_store(self) -> DBMStoreV3:
-        import bsddb3
-
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStoreV3(path, flag="n", open=bsddb3.btopen)
-        return store
-
-    def test_nbytes_stored(self):
-        pass  # not implemented
-
-
 @pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
 @pytest.mark.skipif(have_lmdb is False, reason="needs lmdb")
 class TestArrayWithLMDBStoreV3(TestArrayV3):

zarr/tests/test_hierarchy.py

@@ -1439,27 +1439,6 @@ def create_store():
         return store, None
 
 
-class TestGroupWithDBMStoreBerkeleyDB(TestGroup):
-    @staticmethod
-    def create_store():
-        bsddb3 = pytest.importorskip("bsddb3")
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStore(path, flag="n", open=bsddb3.btopen)
-        return store, None
-
-
-@pytest.mark.skipif(not v3_api_available, reason="V3 is disabled")
-class TestGroupV3WithDBMStoreBerkeleyDB(TestGroupWithDBMStoreBerkeleyDB, TestGroupV3):
-    @staticmethod
-    def create_store():
-        bsddb3 = pytest.importorskip("bsddb3")
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStoreV3(path, flag="n", open=bsddb3.btopen)
-        return store, None
-
-
 class TestGroupWithLMDBStore(TestGroup):
     @staticmethod
     def create_store():

zarr/tests/test_storage.py

@@ -1932,15 +1932,6 @@ def create_store(self, **kwargs):
         return store  # pragma: no cover
 
 
-class TestDBMStoreBerkeleyDB(TestDBMStore):
-    def create_store(self, **kwargs):
-        bsddb3 = pytest.importorskip("bsddb3")
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStore(path, flag="n", open=bsddb3.btopen, write_lock=False, **kwargs)
-        return store
-
-
 class TestLMDBStore(StoreTests):
     def create_store(self, **kwargs):
         pytest.importorskip("lmdb")

zarr/tests/test_storage_v3.py

@@ -53,7 +53,6 @@
 from .test_storage import TestABSStore as _TestABSStore
 from .test_storage import TestConsolidatedMetadataStore as _TestConsolidatedMetadataStore
 from .test_storage import TestDBMStore as _TestDBMStore
-from .test_storage import TestDBMStoreBerkeleyDB as _TestDBMStoreBerkeleyDB
 from .test_storage import TestDBMStoreDumb as _TestDBMStoreDumb
 from .test_storage import TestDBMStoreGnu as _TestDBMStoreGnu
 from .test_storage import TestDBMStoreNDBM as _TestDBMStoreNDBM
@@ -465,15 +464,6 @@ def create_store(self, **kwargs):
         return store  # pragma: no cover
 
 
-class TestDBMStoreV3BerkeleyDB(_TestDBMStoreBerkeleyDB, StoreV3Tests):
-    def create_store(self, **kwargs):
-        bsddb3 = pytest.importorskip("bsddb3")
-        path = mktemp(suffix=".dbm")
-        atexit.register(os.remove, path)
-        store = DBMStoreV3(path, flag="n", open=bsddb3.btopen, write_lock=False, **kwargs)
-        return store
-
-
 class TestLMDBStoreV3(_TestLMDBStore, StoreV3Tests):
     def create_store(self, **kwargs):
         pytest.importorskip("lmdb")