Merge branch 'main' into groupby-aggs-using-numpy-groupies

andersy005 · web-flow · commit 1875fd20a46d · 2021-12-16T07:21:47.000-07:00
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
       - id: isort
   # https://github.com/python/black#version-control-integration
   - repo: https://github.com/psf/black
-    rev: 21.11b1
+    rev: 21.12b0
     hooks:
       - id: black
       - id: black-jupyter
diff --git a/asv_bench/asv.conf.json b/asv_bench/asv.conf.json
@@ -67,6 +67,7 @@
         "distributed": [""],
         "flox": [""],
         "numpy_groupies": [""],
+        "sparse": [""]
     },
 
 
diff --git a/asv_bench/benchmarks/__init__.py b/asv_bench/benchmarks/__init__.py
@@ -22,6 +22,13 @@ def requires_dask():
         raise NotImplementedError()
 
 
+def requires_sparse():
+    try:
+        import sparse  # noqa: F401
+    except ImportError:
+        raise NotImplementedError()
+
+
 def randn(shape, frac_nan=None, chunks=None, seed=0):
     rng = np.random.RandomState(seed)
     if chunks is None:
diff --git a/asv_bench/benchmarks/unstacking.py b/asv_bench/benchmarks/unstacking.py
@@ -1,8 +1,9 @@
 import numpy as np
+import pandas as pd
 
 import xarray as xr
 
-from . import requires_dask
+from . import requires_dask, requires_sparse
 
 
 class Unstacking:
@@ -27,3 +28,37 @@ def setup(self, *args, **kwargs):
         requires_dask()
         super().setup(**kwargs)
         self.da_full = self.da_full.chunk({"flat_dim": 25})
+
+
+class UnstackingSparse(Unstacking):
+    def setup(self, *args, **kwargs):
+        requires_sparse()
+
+        import sparse
+
+        data = sparse.random((500, 1000), random_state=0, fill_value=0)
+        self.da_full = xr.DataArray(data, dims=list("ab")).stack(flat_dim=[...])
+        self.da_missing = self.da_full[:-1]
+
+        mindex = pd.MultiIndex.from_arrays([np.arange(100), np.arange(100)])
+        self.da_eye_2d = xr.DataArray(np.ones((100,)), dims="z", coords={"z": mindex})
+        self.da_eye_3d = xr.DataArray(
+            np.ones((100, 50)),
+            dims=("z", "foo"),
+            coords={"z": mindex, "foo": np.arange(50)},
+        )
+
+    def time_unstack_to_sparse_2d(self):
+        self.da_eye_2d.unstack(sparse=True)
+
+    def time_unstack_to_sparse_3d(self):
+        self.da_eye_3d.unstack(sparse=True)
+
+    def peakmem_unstack_to_sparse_2d(self):
+        self.da_eye_2d.unstack(sparse=True)
+
+    def peakmem_unstack_to_sparse_3d(self):
+        self.da_eye_3d.unstack(sparse=True)
+
+    def time_unstack_pandas_slow(self):
+        pass
diff --git a/ci/requirements/doc.yml b/ci/requirements/doc.yml
@@ -8,7 +8,7 @@ dependencies:
   - bottleneck
   - cartopy
   - cfgrib>=0.9
-  - dask>=2.10
+  - dask-core>=2.30
   - h5netcdf>=0.7.4
   - ipykernel
   - ipython
diff --git a/ci/requirements/environment-windows.yml b/ci/requirements/environment-windows.yml
@@ -8,7 +8,7 @@ dependencies:
   # - cdms2  # Not available on Windows
   # - cfgrib  # Causes Python interpreter crash on Windows: https://github.com/pydata/xarray/pull/3340
   - cftime
-  - dask
+  - dask-core
   - distributed
   - fsspec!=2021.7.0
   - h5netcdf
diff --git a/ci/requirements/environment.yml b/ci/requirements/environment.yml
@@ -10,7 +10,7 @@ dependencies:
   - cdms2
   - cfgrib
   - cftime
-  - dask
+  - dask-core
   - distributed
   - fsspec!=2021.7.0
   - h5netcdf
diff --git a/ci/requirements/py37-min-all-deps.yml b/ci/requirements/py37-min-all-deps.yml
@@ -16,7 +16,7 @@ dependencies:
   - cfgrib=0.9
   - cftime=1.2
   - coveralls
-  - dask=2.30
+  - dask-core=2.30
   - distributed=2.30
   - h5netcdf=0.8
   - h5py=2.10
diff --git a/doc/contributing.rst b/doc/contributing.rst
@@ -318,8 +318,9 @@ to build the docs you need to use the environment file ``ci/requirements/doc.yml
 Building the documentation
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Navigate to your local ``xarray/doc/`` directory in the console and run::
+To build the documentation run::
 
+    cd doc/
     make html
 
 Then you can find the HTML output in the folder ``xarray/doc/_build/html/``.
diff --git a/doc/developers-meeting.rst b/doc/developers-meeting.rst
@@ -0,0 +1,20 @@
+Developers meeting
+------------------
+
+Xarray developers meet bi-weekly every other Wednesday.
+
+The meeting occurs on `Zoom <https://us02web.zoom.us/j/88251613296?pwd=azZsSkU1UWJZTVFKNnhIUVdZcENUZz09>`__.
+
+Notes for the meeting are kept `here <https://hackmd.io/@U4W-olO3TX-hc-cvbjNe4A/xarray-dev-meeting/edit>`__.
+
+There is a `GitHub issue <https://github.com/pydata/xarray/issues/4001>`__ for changes to the meeting.
+
+You can subscribe to this calendar to be notified of changes:
+
+* `Google Calendar <https://calendar.google.com/calendar/embed?src=ucar.edu_2gjd5fuugcj4ol6ij7knj8krn8%40group.calendar.google.com&ctz=America%2FLos_Angeles>`__
+* `iCal <https://calendar.google.com/calendar/ical/ucar.edu_2gjd5fuugcj4ol6ij7knj8krn8%40group.calendar.google.com/public/basic.ics>`__
+
+.. raw:: html
+
+   <iframe id="calendariframe" src="https://calendar.google.com/calendar/embed?ctz=local&amp;src=ucar.edu_2gjd5fuugcj4ol6ij7knj8krn8%40group.calendar.google.com" style="border: 0" width="800" height="600" frameborder="0" scrolling="no"></iframe>
+   <script>document.getElementById("calendariframe").src = document.getElementById("calendariframe").src.replace("ctz=local", "ctz=" + Intl.DateTimeFormat().resolvedOptions().timeZone)</script>
diff --git a/doc/ecosystem.rst b/doc/ecosystem.rst
@@ -13,6 +13,7 @@ Geosciences
 ~~~~~~~~~~~
 
 - `aospy <https://aospy.readthedocs.io>`_: Automated analysis and management of gridded climate data.
+- `argopy <https://github.com/euroargodev/argopy>`_: xarray-based Argo data access, manipulation and visualisation for standard users as well as Argo experts.
 - `climpred <https://climpred.readthedocs.io>`_: Analysis of ensemble forecast models for climate prediction.
 - `geocube <https://corteva.github.io/geocube>`_: Tool to convert geopandas vector data into rasterized xarray data.
 - `GeoWombat <https://github.com/jgrss/geowombat>`_: Utilities for analysis of remotely sensed and gridded raster data at scale (easily tame Landsat, Sentinel, Quickbird, and PlanetScope).
@@ -47,6 +48,7 @@ Geosciences
 - `xESMF <https://pangeo-xesmf.readthedocs.io/>`_: Universal regridder for geospatial data.
 - `xgcm <https://xgcm.readthedocs.io/>`_: Extends the xarray data model to understand finite volume grid cells (common in General Circulation Models) and provides interpolation and difference operations for such grids.
 - `xmitgcm <http://xgcm.readthedocs.io/>`_: a python package for reading `MITgcm <http://mitgcm.org/>`_ binary MDS files into xarray data structures.
+- `xnemogcm <https://github.com/rcaneill/xnemogcm/>`_: a package to read `NEMO <https://nemo-ocean.eu/>`_ output files and add attributes to interface with xgcm.
 
 Machine Learning
 ~~~~~~~~~~~~~~~~
diff --git a/doc/getting-started-guide/faq.rst b/doc/getting-started-guide/faq.rst
@@ -140,12 +140,11 @@ conventions`_. (An exception is serialization to and from netCDF files.)
 
 An implication of this choice is that we do not propagate ``attrs`` through
 most operations unless explicitly flagged (some methods have a ``keep_attrs``
-option, and there is a global flag for setting this to be always True or
-False). Similarly, xarray does not check for conflicts between ``attrs`` when
-combining arrays and datasets, unless explicitly requested with the option
-``compat='identical'``. The guiding principle is that metadata should not be
-allowed to get in the way.
-
+option, and there is a global flag, accessible with :py:func:`xarray.set_options`,
+for setting this to be always True or False). Similarly, xarray does not check
+for conflicts between ``attrs`` when combining arrays and datasets, unless
+explicitly requested with the option ``compat='identical'``. The guiding
+principle is that metadata should not be allowed to get in the way.
 
 What other netCDF related Python libraries should I know about?
 ---------------------------------------------------------------
diff --git a/doc/index.rst b/doc/index.rst
@@ -45,6 +45,7 @@ computing.
    Xarray Internals <internals/index>
    Development Roadmap <roadmap>
    Team <team>
+   Developers Meeting <developers-meeting>
    What’s New <whats-new>
    GitHub repository <https://github.com/pydata/xarray>
 
diff --git a/doc/user-guide/computation.rst b/doc/user-guide/computation.rst
@@ -548,7 +548,7 @@ two gaussian peaks:
 Broadcasting by dimension name
 ==============================
 
-``DataArray`` objects are automatically align themselves ("broadcasting" in
+``DataArray`` objects automatically align themselves ("broadcasting" in
 the numpy parlance) by dimension name instead of axis order. With xarray, you
 do not need to transpose arrays or insert dimensions of length 1 to get array
 operations to work, as commonly done in numpy with :py:func:`numpy.reshape` or
diff --git a/doc/whats-new.rst b/doc/whats-new.rst
@@ -17,23 +17,36 @@ What's New
 
 .. _whats-new.0.20.2:
 
-v0.20.2 (unreleased)
----------------------
+v0.20.2 (9 December 2021)
+-------------------------
 
-New Features
-~~~~~~~~~~~~
+This is a bugfix release to resolve (:issue:`3391`, :issue:`5715`). It also
+includes performance improvements in unstacking to a ``sparse`` array and a
+number of documentation improvements.
+
+Many thanks to the 20 contributors:
+
+Aaron Spring, Alexandre Poux, Deepak Cherian, Enrico Minack, Fabien Maussion,
+Giacomo Caria, Gijom, Guillaume Maze, Illviljan, Joe Hamman, Joseph Hardin, Kai
+Mühlbauer, Matt Henderson, Maximilian Roos, Michael Delgado, Robert Gieseke,
+Sebastian Weigand and Stephan Hoyer.
 
 
 Breaking changes
 ~~~~~~~~~~~~~~~~
+- Use complex nan when interpolating complex values out of bounds by default (instead of real nan) (:pull:`6019`).
+  By `Alexandre Poux <https://github.com/pums974>`_.
 
+Performance
+~~~~~~~~~~~
 
-Deprecations
-~~~~~~~~~~~~
-
+- Significantly faster unstacking to a ``sparse`` array. :pull:`5577`
+  By `Deepak Cherian <https://github.com/dcherian>`_.
 
 Bug fixes
 ~~~~~~~~~
+- :py:func:`xr.map_blocks` and :py:func:`xr.corr` now work when dask is not installed (:issue:`3391`, :issue:`5715`, :pull:`5731`).
+  By `Gijom <https://github.com/Gijom>`_.
 - Fix plot.line crash for data of shape ``(1, N)`` in _title_for_slice on format_item (:pull:`5948`).
   By `Sebastian Weigand <https://github.com/s-weigand>`_.
 - Fix a regression in the removal of duplicate backend entrypoints (:issue:`5944`, :pull:`5959`)
diff --git a/xarray/core/computation.py b/xarray/core/computation.py
@@ -1346,24 +1346,10 @@ def _cov_corr(da_a, da_b, dim=None, ddof=0, method=None):
 
     # 2. Ignore the nans
     valid_values = da_a.notnull() & da_b.notnull()
+    da_a = da_a.where(valid_values)
+    da_b = da_b.where(valid_values)
     valid_count = valid_values.sum(dim) - ddof
 
-    def _get_valid_values(da, other):
-        """
-        Function to lazily mask da_a and da_b
-        following a similar approach to
-        https://github.com/pydata/xarray/pull/4559
-        """
-        missing_vals = np.logical_or(da.isnull(), other.isnull())
-        if missing_vals.any():
-            da = da.where(~missing_vals)
-            return da
-        else:
-            return da
-
-    da_a = da_a.map_blocks(_get_valid_values, args=[da_b])
-    da_b = da_b.map_blocks(_get_valid_values, args=[da_a])
-
     # 3. Detrend along the given dim
     demeaned_da_a = da_a - da_a.mean(dim=dim)
     demeaned_da_b = da_b - da_b.mean(dim=dim)
diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py
@@ -4046,7 +4046,9 @@ def ensure_stackable(val):
 
         return data_array
 
-    def _unstack_once(self, dim: Hashable, fill_value) -> "Dataset":
+    def _unstack_once(
+        self, dim: Hashable, fill_value, sparse: bool = False
+    ) -> "Dataset":
         index = self.get_index(dim)
         index = remove_unused_levels_categories(index)
 
@@ -4062,7 +4064,7 @@ def _unstack_once(self, dim: Hashable, fill_value) -> "Dataset":
                         fill_value_ = fill_value
 
                     variables[name] = var._unstack_once(
-                        index=index, dim=dim, fill_value=fill_value_
+                        index=index, dim=dim, fill_value=fill_value_, sparse=sparse
                     )
                 else:
                     variables[name] = var
@@ -4196,7 +4198,7 @@ def unstack(
         #    Once that is resolved, explicitly exclude pint arrays.
         #    pint doesn't implement `np.full_like` in a way that's
         #    currently compatible.
-        needs_full_reindex = sparse or any(
+        needs_full_reindex = any(
             is_duck_dask_array(v.data)
             or isinstance(v.data, sparse_array_type)
             or not isinstance(v.data, np.ndarray)
@@ -4207,7 +4209,7 @@ def unstack(
             if needs_full_reindex:
                 result = result._unstack_full_reindex(dim, fill_value, sparse)
             else:
-                result = result._unstack_once(dim, fill_value)
+                result = result._unstack_once(dim, fill_value, sparse)
         return result
 
     def update(self, other: "CoercibleMapping") -> "Dataset":
diff --git a/xarray/core/missing.py b/xarray/core/missing.py
@@ -82,9 +82,11 @@ def __init__(self, xi, yi, method="linear", fill_value=None, period=None):
         self._xi = xi
         self._yi = yi
 
+        nan = np.nan if yi.dtype.kind != "c" else np.nan + np.nan * 1j
+
         if fill_value is None:
-            self._left = np.nan
-            self._right = np.nan
+            self._left = nan
+            self._right = nan
         elif isinstance(fill_value, Sequence) and len(fill_value) == 2:
             self._left = fill_value[0]
             self._right = fill_value[1]
@@ -143,10 +145,12 @@ def __init__(
         self.cons_kwargs = kwargs
         self.call_kwargs = {}
 
+        nan = np.nan if yi.dtype.kind != "c" else np.nan + np.nan * 1j
+
         if fill_value is None and method == "linear":
-            fill_value = np.nan, np.nan
+            fill_value = nan, nan
         elif fill_value is None:
-            fill_value = np.nan
+            fill_value = nan
 
         self.f = interp1d(
             xi,
diff --git a/xarray/core/parallel.py b/xarray/core/parallel.py
@@ -23,6 +23,7 @@
 from .alignment import align
 from .dataarray import DataArray
 from .dataset import Dataset
+from .pycompat import is_dask_collection
 
 try:
     import dask
@@ -328,13 +329,13 @@ def _wrapper(
         raise TypeError("kwargs must be a mapping (for example, a dict)")
 
     for value in kwargs.values():
-        if dask.is_dask_collection(value):
+        if is_dask_collection(value):
             raise TypeError(
                 "Cannot pass dask collections in kwargs yet. Please compute or "
                 "load values before passing to map_blocks."
             )
 
-    if not dask.is_dask_collection(obj):
+    if not is_dask_collection(obj):
         return func(obj, *args, **kwargs)
 
     all_args = [obj] + list(args)
diff --git a/xarray/core/pycompat.py b/xarray/core/pycompat.py
@@ -43,15 +43,19 @@ def __init__(self, mod):
         self.available = duck_array_module is not None
 
 
-def is_duck_dask_array(x):
+def is_dask_collection(x):
     if DuckArrayModule("dask").available:
         from dask.base import is_dask_collection
 
-        return is_duck_array(x) and is_dask_collection(x)
+        return is_dask_collection(x)
     else:
         return False
 
 
+def is_duck_dask_array(x):
+    return is_duck_array(x) and is_dask_collection(x)
+
+
 dsk = DuckArrayModule("dask")
 dask_version = dsk.version
 dask_array_type = dsk.type
diff --git a/xarray/core/utils.py b/xarray/core/utils.py
@@ -326,7 +326,6 @@ def is_scalar(value: Any, include_0d: bool = True) -> bool:
             """
             return _is_scalar(value, include_0d)
 
-
 else:
 
     def is_scalar(value: Any, include_0d: bool = True) -> TypeGuard[Hashable]:
diff --git a/xarray/core/variable.py b/xarray/core/variable.py
diff --git a/xarray/plot/facetgrid.py b/xarray/plot/facetgrid.py
diff --git a/xarray/tests/test_computation.py b/xarray/tests/test_computation.py
diff --git a/xarray/tests/test_dataset.py b/xarray/tests/test_dataset.py
diff --git a/xarray/tests/test_interp.py b/xarray/tests/test_interp.py
diff --git a/xarray/tests/test_missing.py b/xarray/tests/test_missing.py
