WIP: Test lowest versions of all required and optional dependencies #3639
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -33,18 +33,18 @@ dependencies = [ | |
| "numpy>=1.24", | ||
| "pandas>=2.0", | ||
| "xarray>=2023.04", | ||
| "netCDF4>=1.7", | ||
|
There was a problem hiding this comment. We discussed making NetCDF an optional dependency before in #429. Maybe we could revisit that discussion? There was a problem hiding this comment. That's a good point. NetCDF4 was needed by xarray when reading and writing netCDF files. As mentioned in #239 (comment), previously, our After we refactor our I've opened PR #3643 to see how the PyGMT relies on the netCDF package. Edit: There are only 14 failures in the Python 3.11 + Ubuntu CI job after removing netCDF entirely (see https://github.com/GenericMappingTools/pygmt/actions/runs/11968416461/job/33367210857?pr=3643) and most of them are caused by There was a problem hiding this comment. There is another error that popped up on Ubuntu-22.04-ARM64 at https://github.com/GenericMappingTools/pygmt/actions/runs/14457975280/job/40545076044?pr=3639#step:7:1079, related to ==================================== ERRORS ====================================
__________________ ERROR at setup of test_clib_read_data_grid __________________
self = CachingFileManager(<class 'netCDF4._netCDF4.Dataset'>, '/home/runner/.gmt/cache/static_earth_relief.nc', mode='r', kwa...r': True, 'diskless': False, 'persist': False, 'format': 'NETCDF4'}, manager_id='26bc1ee9-cb26-4f6f-bcea-ac669a2faa62')
needs_lock = True
def _acquire_with_cache_info(self, needs_lock=True):
"""Acquire a file, returning the file and whether it was cached."""
with self._optional_lock(needs_lock):
try:
> file = self._cache[self._key]
../.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py:210:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <xarray.backends.lru_cache.LRUCache object at 0xff802d7a2b80>
key = [<class 'netCDF4._netCDF4.Dataset'>, ('/home/runner/.gmt/cache/static_earth_relief.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '26bc1ee9-cb26-4f6f-bcea-ac669a2faa62']
def __getitem__(self, key: K) -> V:
# record recent use of the key by moving it to the front of the list
with self._lock:
> value = self._cache[key]
E KeyError: [<class 'netCDF4._netCDF4.Dataset'>, ('/home/runner/.gmt/cache/static_earth_relief.nc',), 'r', (('clobber', True), ('diskless', False), ('format', 'NETCDF4'), ('persist', False)), '26bc1ee9-cb26-4f6f-bcea-ac669a2faa62']
../.venv/lib/python3.11/site-packages/xarray/backends/lru_cache.py:56: KeyError
During handling of the above exception, another exception occurred:
@pytest.fixture(scope="module", name="expected_xrgrid")
def fixture_expected_xrgrid():
"""
The expected xr.DataArray object for the static_earth_relief.nc file.
"""
> return load_dataarray(which("@static_earth_relief.nc"))
../pygmt/tests/test_clib_read_data.py:30:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../pygmt/io.py:44: in load_dataarray
with xr.open_dataarray(filename_or_obj, **kwargs) as dataarray:
../.venv/lib/python3.11/site-packages/xarray/backends/api.py:686: in open_dataarray
dataset = open_dataset(
../.venv/lib/python3.11/site-packages/xarray/backends/api.py:525: in open_dataset
backend_ds = backend.open_dataset(
../.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:588: in open_dataset
store = NetCDF4DataStore.open(
../.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:389: in open
return cls(manager, group=group, mode=mode, lock=lock, autoclose=autoclose)
../.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:336: in __init__
self.format = self.ds.data_model
../.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:398: in ds
return self._acquire()
../.venv/lib/python3.11/site-packages/xarray/backends/netCDF4_.py:392: in _acquire
with self._manager.acquire_context(needs_lock) as root:
../../../../.local/share/uv/python/cpython-3.11.12-linux-aarch64-gnu/lib/python3.11/contextlib.py:137: in __enter__
return next(self.gen)
../.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py:198: in acquire_context
file, cached = self._acquire_with_cache_info(needs_lock)
../.venv/lib/python3.11/site-packages/xarray/backends/file_manager.py:216: in _acquire_with_cache_info
file = self._opener(*self._args, **kwargs)
src/netCDF4/_netCDF4.pyx:2489: in netCDF4._netCDF4.Dataset.__init__
???
src/netCDF4/_netCDF4.pyx:2092: in netCDF4._netCDF4._get_vars
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E RuntimeError: only endian='native' allowed for NETCDF3 files, got 'big' (variable 'lon', group '/')
src/netCDF4/_netCDF4.pyx:4062: RuntimeErrorThis seems to be fixed in Unidata/netcdf4-python#1355, released in netcdf4=1.7.2. So we can either set the minimum version to |
||
| "packaging>=22.0", | ||
| ] | ||
| dynamic = ["version"] | ||
|
|
||
| [project.optional-dependencies] | ||
| all = [ | ||
| "contextily>=1.2", | ||
|
There was a problem hiding this comment. Need to pin to pygmt/pygmt/datasets/tile_map.py Lines 12 to 19 in 97185e8 Xref geopandas/contextily#183, where There was a problem hiding this comment. Good to know that we require contexily>=1.2 |
||
| "geopandas>=0.14", | ||
| "IPython>=8", # 'ipython' is not the correct module name. | ||
| "pyarrow>=16", | ||
|
There was a problem hiding this comment. Pinning to ________ test_to_numpy_pyarrow_array_pyarrow_dtypes_string[string_view] ________
> ???
E KeyError: 'string_view'
pyarrow/types.pxi:5025: KeyError
During handling of the above exception, another exception occurred:
dtype = 'string_view'
@pytest.mark.skipif(not _HAS_PYARROW, reason="pyarrow is not installed")
@pytest.mark.parametrize(
"dtype",
[
None,
"string",
"utf8", # alias for string
"large_string",
"large_utf8", # alias for large_string
"string_view",
],
)
def test_to_numpy_pyarrow_array_pyarrow_dtypes_string(dtype):
"""
Test the _to_numpy function with PyArrow arrays of PyArrow string types.
"""
> array = pa.array(["abc", "defg", "12345"], type=dtype)
../pygmt/tests/test_clib_to_numpy.py:333:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
pyarrow/array.pxi:230: in pyarrow.lib.array
???
pyarrow/types.pxi:5040: in pyarrow.lib.ensure_type
???
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
> ???
E ValueError: No type alias for string_view
pyarrow/types.pxi:5027: ValueErrorCode was added in #3608: pygmt/pygmt/tests/test_clib_to_numpy.py Lines 326 to 336 in 97185e8 The There was a problem hiding this comment. PyArrow v16.0 was released in April, 2024. Instead of pinning pyarrow>=16.0, we can just skip There was a problem hiding this comment. Ok, I just tested older versions from pyarrow v12 to v15, and think we can pin to _________________________________________________________________ test_to_numpy_pyarrow_array_pyarrow_dtypes_date[date64[ms]] __________________________________________________________________
dtype = 'date64[ms]', expected_dtype = 'datetime64[ms]'
@pytest.mark.skipif(not _HAS_PYARROW, reason="pyarrow is not installed")
@pytest.mark.parametrize(
("dtype", "expected_dtype"),
[
pytest.param("date32[day]", "datetime64[D]", id="date32[day]"),
pytest.param("date64[ms]", "datetime64[ms]", id="date64[ms]"),
],
)
def test_to_numpy_pyarrow_array_pyarrow_dtypes_date(dtype, expected_dtype):
"""
Test the _to_numpy function with PyArrow arrays of PyArrow date types.
date32[day] and date64[ms] are stored as 32-bit and 64-bit integers, respectively,
representing the number of days and milliseconds since the UNIX epoch (1970-01-01).
Here we explicitly check the dtype and date unit of the result.
"""
data = [
date(2024, 1, 1),
datetime(2024, 1, 2),
datetime(2024, 1, 3),
]
array = pa.array(data, type=dtype)
result = _to_numpy(array)
_check_result(result, np.datetime64)
> assert result.dtype == expected_dtype # Explicitly check the date unit.
E AssertionError: assert dtype('<M8[D]') == 'datetime64[ms]'
E + where dtype('<M8[D]') = array(['2024-01-01', '2024-01-02', '2024-01-03'], dtype='datetime64[D]').dtype
../pygmt/tests/test_clib_to_numpy.py:364: AssertionErrorThe main change appears to be in apache/arrow#33321, when pyarrow supported preserving the datetime64 temporal resolution. Maybe we can keep the |
||
| "rioxarray>=0.14", | ||
| ] | ||
|
|
||
| [project.urls] | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Using
--resolution lowest-directfor now, because--resolution lowestgrabs many other transitive dependencies that may be hard to install (e.g. missing wheels for newer Python versions, so requires compilation from source). We could switch to--resolution lowestonce the Python ecosystem does lower bound pins a bit more thoroughly (which may take years).