bump minimum versions, drop py38 (#7461)

* bump minimum versions, drop py38

* bump mypy action

* bump pyupgrade

* update whats new

* audit version checks

* remove GenericAlias

* fix a few broken tests

* stop requiring warning for invalid netcdf

* remove extra requires

* bump pandas

* cleanup one more pytest skip

* Apply suggestions from code review

Co-authored-by: Justus Magin <keewis@users.noreply.github.com>

* Update requirements.txt

Co-authored-by: Justus Magin <keewis@users.noreply.github.com>

Co-authored-by: Justus Magin <keewis@users.noreply.github.com>

Author: jhamman

.github/workflows/ci-additional.yaml

@@ -134,18 +134,16 @@ jobs:
           name: codecov-umbrella
           fail_ci_if_error: false
 
-  mypy38:
-    name: Mypy 3.8
+  mypy39:
+    name: Mypy 3.9
     runs-on: "ubuntu-latest"
     needs: detect-ci-trigger
-    # temporarily skipping due to https://github.com/pydata/xarray/issues/6551
-    if: needs.detect-ci-trigger.outputs.triggered == 'false'
     defaults:
       run:
         shell: bash -l {0}
     env:
       CONDA_ENV_FILE: ci/requirements/environment.yml
-      PYTHON_VERSION: "3.8"
+      PYTHON_VERSION: "3.9"
 
     steps:
       - uses: actions/checkout@v3
@@ -185,7 +183,7 @@ jobs:
         uses: codecov/codecov-action@v3.1.1
         with:
           file: mypy_report/cobertura.xml
-          flags: mypy38
+          flags: mypy39
           env_vars: PYTHON_VERSION
           name: codecov-umbrella
           fail_ci_if_error: false

.github/workflows/ci.yaml

@@ -42,15 +42,15 @@ jobs:
       matrix:
         os: ["ubuntu-latest", "macos-latest", "windows-latest"]
         # Bookend python versions
-        python-version: ["3.8", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
         env: [""]
         include:
           # Minimum python version:
           - env: "bare-minimum"
-            python-version: "3.8"
+            python-version: "3.9"
             os: ubuntu-latest
           - env: "min-all-deps"
-            python-version: "3.8"
+            python-version: "3.9"
             os: ubuntu-latest
           # Latest python version:
           - env: "all-but-dask"

.github/workflows/pypi-release.yaml

@@ -18,7 +18,7 @@ jobs:
       - uses: actions/setup-python@v4
         name: Install Python
         with:
-          python-version: 3.8
+          python-version: "3.11"
 
       - name: Install dependencies
         run: |
@@ -53,7 +53,7 @@ jobs:
       - uses: actions/setup-python@v4
         name: Install Python
         with:
-          python-version: 3.8
+          python-version: "3.11"
       - uses: actions/download-artifact@v3
         with:
           name: releases

.pre-commit-config.yaml

@@ -29,7 +29,7 @@ repos:
     hooks:
       - id: pyupgrade
         args:
-          - "--py38-plus"
+          - "--py39-plus"
   # https://github.com/python/black#version-control-integration
   - repo: https://github.com/psf/black
     rev: 22.12.0

asv_bench/asv.conf.json

@@ -40,7 +40,7 @@
 
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
-    "pythons": ["3.8"],
+    "pythons": ["3.10"],
 
     // The matrix of dependencies to test.  Each key is the name of a
     // package (in PyPI) and the values are version numbers.  An empty

ci/min_deps_check.py

@@ -1,11 +1,13 @@
+#!/usr/bin/env python
 """Fetch from conda database all available versions of the xarray dependencies and their
 publication date. Compare it against requirements/py37-min-all-deps.yml to verify the
 policy on obsolete dependencies is being followed. Print a pretty report :)
 """
 import itertools
 import sys
+from collections.abc import Iterator
 from datetime import datetime
-from typing import Dict, Iterator, Optional, Tuple
+from typing import Optional
 
 import conda.api  # type: ignore[import]
 import yaml
@@ -29,7 +31,7 @@
 
 POLICY_MONTHS = {"python": 24, "numpy": 18}
 POLICY_MONTHS_DEFAULT = 12
-POLICY_OVERRIDE: Dict[str, Tuple[int, int]] = {}
+POLICY_OVERRIDE: dict[str, tuple[int, int]] = {}
 errors = []
 
 
@@ -43,7 +45,7 @@ def warning(msg: str) -> None:
     print("WARNING:", msg)
 
 
-def parse_requirements(fname) -> Iterator[Tuple[str, int, int, Optional[int]]]:
+def parse_requirements(fname) -> Iterator[tuple[str, int, int, Optional[int]]]:
     """Load requirements/py37-min-all-deps.yml
 
     Yield (package name, major version, minor version, [patch version])
@@ -75,7 +77,7 @@ def parse_requirements(fname) -> Iterator[Tuple[str, int, int, Optional[int]]]:
             raise ValueError("expected major.minor or major.minor.patch: " + row)
 
 
-def query_conda(pkg: str) -> Dict[Tuple[int, int], datetime]:
+def query_conda(pkg: str) -> dict[tuple[int, int], datetime]:
     """Query the conda repository for a specific package
 
     Return map of {(major version, minor version): publication date}
@@ -115,7 +117,7 @@ def metadata(entry):
 
 def process_pkg(
     pkg: str, req_major: int, req_minor: int, req_patch: Optional[int]
-) -> Tuple[str, str, str, str, str, str]:
+) -> tuple[str, str, str, str, str, str]:
     """Compare package version from requirements file to available versions in conda.
     Return row to build pandas dataframe:
 

ci/requirements/bare-minimum.yml

@@ -3,13 +3,13 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - python=3.8
+  - python=3.9
   - coveralls
   - pip
   - pytest
   - pytest-cov
   - pytest-env
   - pytest-xdist
-  - numpy=1.20
+  - numpy=1.21
   - packaging=21.3
-  - pandas=1.3
+  - pandas=1.4

ci/requirements/doc.yml

@@ -8,8 +8,8 @@ dependencies:
   - bottleneck
   - cartopy
   - cfgrib>=0.9
-  - dask-core>=2.30
-  - h5netcdf>=0.7.4
+  - dask-core>=2022.1
+  - h5netcdf>=0.13
   - ipykernel
   - ipython
   - iris>=2.3
@@ -18,9 +18,9 @@ dependencies:
   - nbsphinx
   - netcdf4>=1.5
   - numba
-  - numpy>=1.20,<1.24
-  - packaging>=21.0
-  - pandas>=1.3
+  - numpy>=1.21,<1.24
+  - packaging>=21.3
+  - pandas>=1.4
   - pooch
   - pip
   - pydata-sphinx-theme>=0.4.3
@@ -35,7 +35,7 @@ dependencies:
   - sphinx-copybutton
   - sphinx-design
   - sphinx!=4.4.0
-  - zarr>=2.4
+  - zarr>=2.10
   - pip:
       - sphinxext-rediraffe
       - sphinxext-opengraph

ci/requirements/min-all-deps.yml

@@ -7,35 +7,35 @@ dependencies:
   # Run ci/min_deps_check.py to verify that this file respects the policy.
   # When upgrading python, numpy, or pandas, must also change
   # doc/user-guide/installing.rst, doc/user-guide/plotting.rst and setup.py.
-  - python=3.8
+  - python=3.9
   - boto3=1.20
   - bottleneck=1.3
   - cartopy=0.20
   - cdms2=3.1
   - cfgrib=0.9
   - cftime=1.5
   - coveralls
-  - dask-core=2021.11
-  - distributed=2021.11
+  - dask-core=2022.1
+  - distributed=2022.1
   - flox=0.5
-  - h5netcdf=0.11
+  - h5netcdf=0.13
   # h5py and hdf5 tend to cause conflicts
   # for e.g. hdf5 1.12 conflicts with h5py=3.1
   # prioritize bumping other packages instead
   - h5py=3.6
   - hdf5=1.12
   - hypothesis
   - iris=3.1
-  - lxml=4.6  # Optional dep of pydap
+  - lxml=4.7  # Optional dep of pydap
   - matplotlib-base=3.5
   - nc-time-axis=1.4
   # netcdf follows a 1.major.minor[.patch] convention
   # (see https://github.com/Unidata/netcdf4-python/issues/1090)
   - netcdf4=1.5.7
-  - numba=0.54
-  - numpy=1.20
+  - numba=0.55
+  - numpy=1.21
   - packaging=21.3
-  - pandas=1.3
+  - pandas=1.4
   - pint=0.18
   - pip
   - pseudonetcdf=3.2

doc/contributing.rst

@@ -154,7 +154,7 @@ We'll now kick off a two-step process:
 .. code-block:: sh
 
    # Create and activate the build environment
-   conda create -c conda-forge -n xarray-tests python=3.8
+   conda create -c conda-forge -n xarray-tests python=3.10
 
    # This is for Linux and MacOS
    conda env update -f ci/requirements/environment.yml
@@ -571,9 +571,9 @@ A test run of this yields
 
    ((xarray) $ pytest test_cool_feature.py -v
     =============================== test session starts ================================
-    platform darwin -- Python 3.6.4, pytest-3.2.1, py-1.4.34, pluggy-0.4.0 --
-    cachedir: ../../.cache
-    plugins: cov-2.5.1, hypothesis-3.23.0
+    platform darwin -- Python 3.10.6, pytest-7.2.0, pluggy-1.0.0 --
+    cachedir: .pytest_cache
+    plugins: hypothesis-6.56.3, cov-4.0.0
     collected 11 items
 
     test_cool_feature.py::test_dtypes[int8] PASSED
@@ -599,7 +599,9 @@ which match ``int8``.
 
    ((xarray) bash-3.2$ pytest  test_cool_feature.py  -v -k int8
    =========================== test session starts ===========================
-   platform darwin -- Python 3.6.2, pytest-3.2.1, py-1.4.31, pluggy-0.4.0
+    platform darwin -- Python 3.10.6, pytest-7.2.0, pluggy-1.0.0 --
+    cachedir: .pytest_cache
+    plugins: hypothesis-6.56.3, cov-4.0.0
    collected 11 items
 
    test_cool_feature.py::test_dtypes[int8] PASSED
@@ -645,8 +647,7 @@ Performance matters and it is worth considering whether your code has introduced
 performance regressions.  *xarray* is starting to write a suite of benchmarking tests
 using `asv <https://github.com/spacetelescope/asv>`__
 to enable easy monitoring of the performance of critical *xarray* operations.
-These benchmarks are all found in the ``xarray/asv_bench`` directory.  asv
-supports both python2 and python3.
+These benchmarks are all found in the ``xarray/asv_bench`` directory.
 
 To use all features of asv, you will need either ``conda`` or
 ``virtualenv``. For more details please check the `asv installation
@@ -699,7 +700,7 @@ environment by::
 
 or, to use a specific Python interpreter,::
 
-    asv run -e -E existing:python3.6
+    asv run -e -E existing:python3.10
 
 This will display stderr from the benchmarks, and use your local
 ``python`` that comes from your ``$PATH``.