TST Extend tests for scipy.sparse/*array in sklearn/neighbors/tests/test_neighbors (scikit-learn#27250)

Author: Charlie-XIAO

doc/whats_new/v1.4.rst

@@ -77,20 +77,32 @@ and classes are impacted:
 
 **Functions:**
 
+- :func:`cluster.compute_optics_graph` in :pr:`27250` by
+  :user:`Yao Xiao <Charlie-XIAO>`;
 - :func:`decomposition.non_negative_factorization` in :pr:`27100` by
   :user:`Isaac Virshup <ivirshup>`;
+- :func:`manifold.trustworthiness` in :pr:`27250` by :user:`Yao Xiao <Charlie-XIAO>`;
 - :func:`metrics.f_regression` in :pr:`27239` by :user:`Yaroslav Korobko <Tialo>`;
+- :func:`metrics.pairwise_distances` in :pr:`27250` by :user:`Yao Xiao <Charlie-XIAO>`;
+- :func:`metrics.pairwise_distances_chunked` in :pr:`27250` by
+  :user:`Yao Xiao <Charlie-XIAO>`;
+- :func:`metrics.pairwise.pairwise_kernels` in :pr:`27250` by
+  :user:`Yao Xiao <Charlie-XIAO>`;
 - :func:`metrics.r_regression` in :pr:`27239` by :user:`Yaroslav Korobko <Tialo>`;
 - :func:`sklearn.utils.multiclass.type_of_target` in :pr:`27274` by
   :user:`Yao Xiao <Charlie-XIAO>`.
 
 **Classes:**
 
+- :class:`cluster.HDBSCAN` in :pr:`27250` by :user:`Yao Xiao <Charlie-XIAO>`;
+- :class:`cluster.OPTICS` in :pr:`27250` by :user:`Yao Xiao <Charlie-XIAO>`;
 - :class:`decomposition.NMF` in :pr:`27100` by :user:`Isaac Virshup <ivirshup>`;
 - :class:`decomposition.MiniBatchNMF` in :pr:`27100` by
   :user:`Isaac Virshup <ivirshup>`;
 - :class:`feature_extraction.text.TfidfTransformer` in :pr:`27219` by
   :user:`Yao Xiao <Charlie-XIAO>`;
+- :class:`cluster.Isomap` in :pr:`27250` by :user:`Yao Xiao <Charlie-XIAO>`;
+- :class:`manifold.TSNE` in :pr:`27250` by :user:`Yao Xiao <Charlie-XIAO>`;
 - :class:`impute.SimpleImputer` in :pr:`27277` by :user:`Yao Xiao <Charlie-XIAO>`;
 - :class:`impute.IterativeImputer` in :pr:`27277` by :user:`Yao Xiao <Charlie-XIAO>`;
 - :class:`impute.KNNImputer` in :pr:`27277` by :user:`Yao Xiao <Charlie-XIAO>`;

sklearn/metrics/pairwise.py

@@ -1826,7 +1826,11 @@ def _pairwise_callable(X, Y, metric, force_all_finite=True, **kwds):
         out = np.zeros((X.shape[0], Y.shape[0]), dtype="float")
         iterator = itertools.combinations(range(X.shape[0]), 2)
         for i, j in iterator:
-            out[i, j] = metric(X[i], Y[j], **kwds)
+            # scipy has not yet implemented 1D sparse slices; once implemented this can
+            # be removed and `arr[ind]` can be simply used.
+            x = X[[i], :] if issparse(X) else X[i]
+            y = Y[[j], :] if issparse(Y) else Y[j]
+            out[i, j] = metric(x, y, **kwds)
 
         # Make symmetric
         # NB: out += out.T will produce incorrect results
@@ -1835,15 +1839,21 @@ def _pairwise_callable(X, Y, metric, force_all_finite=True, **kwds):
         # Calculate diagonal
         # NB: nonzero diagonals are allowed for both metrics and kernels
         for i in range(X.shape[0]):
-            x = X[i]
+            # scipy has not yet implemented 1D sparse slices; once implemented this can
+            # be removed and `arr[ind]` can be simply used.
+            x = X[[i], :] if issparse(X) else X[i]
             out[i, i] = metric(x, x, **kwds)
 
     else:
         # Calculate all cells
         out = np.empty((X.shape[0], Y.shape[0]), dtype="float")
         iterator = itertools.product(range(X.shape[0]), range(Y.shape[0]))
         for i, j in iterator:
-            out[i, j] = metric(X[i], Y[j], **kwds)
+            # scipy has not yet implemented 1D sparse slices; once implemented this can
+            # be removed and `arr[ind]` can be simply used.
+            x = X[[i], :] if issparse(X) else X[i]
+            y = Y[[j], :] if issparse(Y) else Y[j]
+            out[i, j] = metric(x, y, **kwds)
 
     return out
 

sklearn/neighbors/tests/test_neighbors.py

@@ -5,16 +5,7 @@
 import joblib
 import numpy as np
 import pytest
-from scipy.sparse import (
-    bsr_matrix,
-    coo_matrix,
-    csc_matrix,
-    csr_matrix,
-    dia_matrix,
-    dok_matrix,
-    issparse,
-    lil_matrix,
-)
+from scipy.sparse import issparse
 
 from sklearn import (
     config_context,
@@ -49,7 +40,17 @@
     assert_array_equal,
     ignore_warnings,
 )
-from sklearn.utils.fixes import parse_version, sp_version
+from sklearn.utils.fixes import (
+    BSR_CONTAINERS,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DIA_CONTAINERS,
+    DOK_CONTAINERS,
+    LIL_CONTAINERS,
+    parse_version,
+    sp_version,
+)
 from sklearn.utils.validation import check_random_state
 
 rng = np.random.RandomState(0)
@@ -65,7 +66,14 @@
 digits.data = digits.data[perm]
 digits.target = digits.target[perm]
 
-SPARSE_TYPES = (bsr_matrix, coo_matrix, csc_matrix, csr_matrix, dok_matrix, lil_matrix)
+SPARSE_TYPES = tuple(
+    BSR_CONTAINERS
+    + COO_CONTAINERS
+    + CSC_CONTAINERS
+    + CSR_CONTAINERS
+    + DOK_CONTAINERS
+    + LIL_CONTAINERS
+)
 SPARSE_OR_DENSE = SPARSE_TYPES + (np.asarray,)
 
 ALGORITHMS = ("ball_tree", "brute", "kd_tree", "auto")
@@ -460,35 +468,37 @@ def make_train_test(X_train, X_test):
     check_precomputed(make_train_test, estimators)
 
 
-def test_is_sorted_by_data():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_is_sorted_by_data(csr_container):
     # Test that _is_sorted_by_data works as expected. In CSR sparse matrix,
     # entries in each row can be sorted by indices, by data, or unsorted.
     # _is_sorted_by_data should return True when entries are sorted by data,
     # and False in all other cases.
 
     # Test with sorted 1D array
-    X = csr_matrix(np.arange(10))
+    X = csr_container(np.arange(10))
     assert _is_sorted_by_data(X)
     # Test with unsorted 1D array
     X[0, 2] = 5
     assert not _is_sorted_by_data(X)
 
     # Test when the data is sorted in each sample, but not necessarily
     # between samples
-    X = csr_matrix([[0, 1, 2], [3, 0, 0], [3, 4, 0], [1, 0, 2]])
+    X = csr_container([[0, 1, 2], [3, 0, 0], [3, 4, 0], [1, 0, 2]])
     assert _is_sorted_by_data(X)
 
     # Test with duplicates entries in X.indptr
     data, indices, indptr = [0, 4, 2, 2], [0, 1, 1, 1], [0, 2, 2, 4]
-    X = csr_matrix((data, indices, indptr), shape=(3, 3))
+    X = csr_container((data, indices, indptr), shape=(3, 3))
     assert _is_sorted_by_data(X)
 
 
 @pytest.mark.filterwarnings("ignore:EfficiencyWarning")
 @pytest.mark.parametrize("function", [sort_graph_by_row_values, _check_precomputed])
-def test_sort_graph_by_row_values(function):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sort_graph_by_row_values(function, csr_container):
     # Test that sort_graph_by_row_values returns a graph sorted by row values
-    X = csr_matrix(np.abs(np.random.RandomState(42).randn(10, 10)))
+    X = csr_container(np.abs(np.random.RandomState(42).randn(10, 10)))
     assert not _is_sorted_by_data(X)
     Xt = function(X)
     assert _is_sorted_by_data(Xt)
@@ -497,16 +507,17 @@ def test_sort_graph_by_row_values(function):
     mask = np.random.RandomState(42).randint(2, size=(10, 10))
     X = X.toarray()
     X[mask == 1] = 0
-    X = csr_matrix(X)
+    X = csr_container(X)
     assert not _is_sorted_by_data(X)
     Xt = function(X)
     assert _is_sorted_by_data(Xt)
 
 
 @pytest.mark.filterwarnings("ignore:EfficiencyWarning")
-def test_sort_graph_by_row_values_copy():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sort_graph_by_row_values_copy(csr_container):
     # Test if the sorting is done inplace if X is CSR, so that Xt is X.
-    X_ = csr_matrix(np.abs(np.random.RandomState(42).randn(10, 10)))
+    X_ = csr_container(np.abs(np.random.RandomState(42).randn(10, 10)))
     assert not _is_sorted_by_data(X_)
 
     # sort_graph_by_row_values is done inplace if copy=False
@@ -531,9 +542,10 @@ def test_sort_graph_by_row_values_copy():
         sort_graph_by_row_values(X.tocsc(), copy=False)
 
 
-def test_sort_graph_by_row_values_warning():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sort_graph_by_row_values_warning(csr_container):
     # Test that the parameter warn_when_not_sorted works as expected.
-    X = csr_matrix(np.abs(np.random.RandomState(42).randn(10, 10)))
+    X = csr_container(np.abs(np.random.RandomState(42).randn(10, 10)))
     assert not _is_sorted_by_data(X)
 
     # warning
@@ -550,36 +562,39 @@ def test_sort_graph_by_row_values_warning():
         sort_graph_by_row_values(X, copy=True, warn_when_not_sorted=False)
 
 
-@pytest.mark.parametrize("format", [dok_matrix, bsr_matrix, dia_matrix])
-def test_sort_graph_by_row_values_bad_sparse_format(format):
+@pytest.mark.parametrize(
+    "sparse_container", DOK_CONTAINERS + BSR_CONTAINERS + DIA_CONTAINERS
+)
+def test_sort_graph_by_row_values_bad_sparse_format(sparse_container):
     # Test that sort_graph_by_row_values and _check_precomputed error on bad formats
-    X = format(np.abs(np.random.RandomState(42).randn(10, 10)))
+    X = sparse_container(np.abs(np.random.RandomState(42).randn(10, 10)))
     with pytest.raises(TypeError, match="format is not supported"):
         sort_graph_by_row_values(X)
     with pytest.raises(TypeError, match="format is not supported"):
         _check_precomputed(X)
 
 
 @pytest.mark.filterwarnings("ignore:EfficiencyWarning")
-def test_precomputed_sparse_invalid():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_precomputed_sparse_invalid(csr_container):
     dist = np.array([[0.0, 2.0, 1.0], [2.0, 0.0, 3.0], [1.0, 3.0, 0.0]])
-    dist_csr = csr_matrix(dist)
+    dist_csr = csr_container(dist)
     neigh = neighbors.NearestNeighbors(n_neighbors=1, metric="precomputed")
     neigh.fit(dist_csr)
     neigh.kneighbors(None, n_neighbors=1)
     neigh.kneighbors(np.array([[0.0, 0.0, 0.0]]), n_neighbors=2)
 
     # Ensures enough number of nearest neighbors
     dist = np.array([[0.0, 2.0, 0.0], [2.0, 0.0, 3.0], [0.0, 3.0, 0.0]])
-    dist_csr = csr_matrix(dist)
+    dist_csr = csr_container(dist)
     neigh.fit(dist_csr)
     msg = "2 neighbors per samples are required, but some samples have only 1"
     with pytest.raises(ValueError, match=msg):
         neigh.kneighbors(None, n_neighbors=1)
 
     # Checks error with inconsistent distance matrix
     dist = np.array([[5.0, 2.0, 1.0], [-2.0, 0.0, 3.0], [1.0, 3.0, 0.0]])
-    dist_csr = csr_matrix(dist)
+    dist_csr = csr_container(dist)
     msg = "Negative values in data passed to precomputed distance matrix."
     with pytest.raises(ValueError, match=msg):
         neigh.kneighbors(dist_csr, n_neighbors=1)
@@ -995,12 +1010,13 @@ def test_radius_neighbors_boundary_handling():
         assert_array_equal(results[0], [0, 1])
 
 
-def test_radius_neighbors_returns_array_of_objects():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_radius_neighbors_returns_array_of_objects(csr_container):
     # check that we can pass precomputed distances to
     # NearestNeighbors.radius_neighbors()
     # non-regression test for
     # https://github.com/scikit-learn/scikit-learn/issues/16036
-    X = csr_matrix(np.ones((4, 4)))
+    X = csr_container(np.ones((4, 4)))
     X.setdiag([0, 0, 0, 0])
 
     nbrs = neighbors.NearestNeighbors(
@@ -1371,7 +1387,7 @@ def test_kneighbors_regressor_sparse(
             assert np.mean(knn.predict(X2).round() == y) > 0.95
 
             X2_pre = sparsev(pairwise_distances(X, metric="euclidean"))
-            if sparsev in {dok_matrix, bsr_matrix}:
+            if sparsev in DOK_CONTAINERS + BSR_CONTAINERS:
                 msg = "not supported due to its handling of explicit zeros"
                 with pytest.raises(TypeError, match=msg):
                     knn_pre.predict(X2_pre)
@@ -1453,12 +1469,13 @@ def test_kneighbors_graph():
 
 @pytest.mark.parametrize("n_neighbors", [1, 2, 3])
 @pytest.mark.parametrize("mode", ["connectivity", "distance"])
-def test_kneighbors_graph_sparse(n_neighbors, mode, seed=36):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_kneighbors_graph_sparse(n_neighbors, mode, csr_container, seed=36):
     # Test kneighbors_graph to build the k-Nearest Neighbor graph
     # for sparse input.
     rng = np.random.RandomState(seed)
     X = rng.randn(10, 10)
-    Xcsr = csr_matrix(X)
+    Xcsr = csr_container(X)
 
     assert_allclose(
         neighbors.kneighbors_graph(X, n_neighbors, mode=mode).toarray(),
@@ -1481,12 +1498,13 @@ def test_radius_neighbors_graph():
 
 @pytest.mark.parametrize("n_neighbors", [1, 2, 3])
 @pytest.mark.parametrize("mode", ["connectivity", "distance"])
-def test_radius_neighbors_graph_sparse(n_neighbors, mode, seed=36):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_radius_neighbors_graph_sparse(n_neighbors, mode, csr_container, seed=36):
     # Test radius_neighbors_graph to build the Nearest Neighbor graph
     # for sparse input.
     rng = np.random.RandomState(seed)
     X = rng.randn(10, 10)
-    Xcsr = csr_matrix(X)
+    Xcsr = csr_container(X)
 
     assert_allclose(
         neighbors.radius_neighbors_graph(X, n_neighbors, mode=mode).toarray(),
@@ -1503,11 +1521,12 @@ def test_radius_neighbors_graph_sparse(n_neighbors, mode, seed=36):
         neighbors.RadiusNeighborsRegressor,
     ],
 )
-def test_neighbors_validate_parameters(Estimator):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_neighbors_validate_parameters(Estimator, csr_container):
     """Additional parameter validation for *Neighbors* estimators not covered by common
     validation."""
     X = rng.random_sample((10, 2))
-    Xsparse = csr_matrix(X)
+    Xsparse = csr_container(X)
     X3 = rng.random_sample((10, 3))
     y = np.ones(10)
 
@@ -1759,13 +1778,14 @@ def custom_metric(x1, x2):
 @pytest.mark.parametrize(
     "metric", neighbors.VALID_METRICS["brute"] + DISTANCE_METRIC_OBJS
 )
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
 def test_valid_brute_metric_for_auto_algorithm(
-    global_dtype, metric, n_samples=20, n_features=12
+    global_dtype, metric, csr_container, n_samples=20, n_features=12
 ):
     metric = _parse_metric(metric, global_dtype)
 
     X = rng.rand(n_samples, n_features).astype(global_dtype, copy=False)
-    Xcsr = csr_matrix(X)
+    Xcsr = csr_container(X)
 
     metric_params_list = _generate_test_params_for(metric, n_features)
 
@@ -1811,7 +1831,8 @@ def test_metric_params_interface():
         est.fit(X, y)
 
 
-def test_predict_sparse_ball_kd_tree():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_predict_sparse_ball_kd_tree(csr_container):
     rng = np.random.RandomState(0)
     X = rng.rand(5, 5)
     y = rng.randint(0, 2, 5)
@@ -1820,7 +1841,7 @@ def test_predict_sparse_ball_kd_tree():
     for model in [nbrs1, nbrs2]:
         model.fit(X, y)
         with pytest.raises(ValueError):
-            model.predict(csr_matrix(X))
+            model.predict(csr_container(X))
 
 
 def test_non_euclidean_kneighbors():
@@ -2073,16 +2094,17 @@ def test_dtype_convert():
     assert_array_equal(result, y)
 
 
-def test_sparse_metric_callable():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_metric_callable(csr_container):
     def sparse_metric(x, y):  # Metric accepting sparse matrix input (only)
         assert issparse(x) and issparse(y)
         return x.dot(y.T).toarray().item()
 
-    X = csr_matrix(
+    X = csr_container(
         [[1, 1, 1, 1, 1], [1, 0, 1, 0, 1], [0, 0, 1, 0, 0]]  # Population matrix
     )
 
-    Y = csr_matrix([[1, 1, 0, 1, 1], [1, 0, 0, 0, 1]])  # Query matrix
+    Y = csr_container([[1, 1, 0, 1, 1], [1, 0, 0, 0, 1]])  # Query matrix
 
     nn = neighbors.NearestNeighbors(
         algorithm="brute", n_neighbors=2, metric=sparse_metric

sklearn/utils/fixes.py

@@ -37,6 +37,7 @@
 LIL_CONTAINERS = [scipy.sparse.lil_matrix]
 DOK_CONTAINERS = [scipy.sparse.dok_matrix]
 BSR_CONTAINERS = [scipy.sparse.bsr_matrix]
+DIA_CONTAINERS = [scipy.sparse.dia_matrix]
 
 if parse_version(scipy.__version__) >= parse_version("1.8"):
     # Sparse Arrays have been added in SciPy 1.8
@@ -49,6 +50,7 @@
     LIL_CONTAINERS.append(scipy.sparse.lil_array)
     DOK_CONTAINERS.append(scipy.sparse.dok_array)
     BSR_CONTAINERS.append(scipy.sparse.bsr_array)
+    DIA_CONTAINERS.append(scipy.sparse.dia_array)
 
 try:
     from scipy.optimize._linesearch import line_search_wolfe1, line_search_wolfe2