TST Extend tests for scipy.sparse.*array in test_pairwise.py (scikit-learn#27288)

Author: StefanieSenger

sklearn/metrics/tests/test_pairwise.py

@@ -3,7 +3,7 @@
 
 import numpy as np
 from numpy import linalg
-from scipy.sparse import csr_matrix, dok_matrix, issparse
+from scipy.sparse import issparse
 from scipy.spatial.distance import (
     cdist,
     cityblock,
@@ -62,11 +62,19 @@
     assert_array_equal,
     ignore_warnings,
 )
-from sklearn.utils.fixes import parse_version, sp_version
+from sklearn.utils.fixes import (
+    BSR_CONTAINERS,
+    COO_CONTAINERS,
+    CSC_CONTAINERS,
+    CSR_CONTAINERS,
+    DOK_CONTAINERS,
+    parse_version,
+    sp_version,
+)
 from sklearn.utils.parallel import Parallel, delayed
 
 
-def test_pairwise_distances(global_dtype):
+def test_pairwise_distances_for_dense_data(global_dtype):
     # Test the pairwise_distance helper function.
     rng = np.random.RandomState(0)
 
@@ -144,10 +152,23 @@ def test_pairwise_distances(global_dtype):
     assert S.shape[1] == Y.shape[0]
     assert_allclose(S, S2)
 
+
+@pytest.mark.parametrize("coo_container", COO_CONTAINERS)
+@pytest.mark.parametrize("csc_container", CSC_CONTAINERS)
+@pytest.mark.parametrize("bsr_container", BSR_CONTAINERS)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_distances_for_sparse_data(
+    coo_container, csc_container, bsr_container, csr_container, global_dtype
+):
+    # Test the pairwise_distance helper function.
+    rng = np.random.RandomState(0)
+    X = rng.random_sample((5, 4)).astype(global_dtype, copy=False)
+    Y = rng.random_sample((2, 4)).astype(global_dtype, copy=False)
+
     # Test with sparse X and Y,
     # currently only supported for Euclidean, L1 and cosine.
-    X_sparse = csr_matrix(X)
-    Y_sparse = csr_matrix(Y)
+    X_sparse = csr_container(X)
+    Y_sparse = csr_container(Y)
 
     S = pairwise_distances(X_sparse, Y_sparse, metric="euclidean")
     S2 = euclidean_distances(X_sparse, Y_sparse)
@@ -159,8 +180,8 @@ def test_pairwise_distances(global_dtype):
     assert_allclose(S, S2)
     assert S.dtype == S2.dtype == global_dtype
 
-    S = pairwise_distances(X_sparse, Y_sparse.tocsc(), metric="manhattan")
-    S2 = manhattan_distances(X_sparse.tobsr(), Y_sparse.tocoo())
+    S = pairwise_distances(X_sparse, csc_container(Y), metric="manhattan")
+    S2 = manhattan_distances(bsr_container(X), coo_container(Y))
     assert_allclose(S, S2)
     if global_dtype == np.float64:
         assert S.dtype == S2.dtype == global_dtype
@@ -368,7 +389,8 @@ def test_pairwise_callable_nonstrict_metric():
     "metric",
     ["rbf", "laplacian", "sigmoid", "polynomial", "linear", "chi2", "additive_chi2"],
 )
-def test_pairwise_kernels(metric):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_kernels(metric, csr_container):
     # Test the pairwise_kernels helper function.
 
     rng = np.random.RandomState(0)
@@ -390,8 +412,8 @@ def test_pairwise_kernels(metric):
     assert_allclose(K1, K2)
 
     # Test with sparse X and Y
-    X_sparse = csr_matrix(X)
-    Y_sparse = csr_matrix(Y)
+    X_sparse = csr_container(X)
+    Y_sparse = csr_container(Y)
     if metric in ["chi2", "additive_chi2"]:
         # these don't support sparse matrices yet
         return
@@ -432,7 +454,8 @@ def test_pairwise_kernels_filter_param():
 
 
 @pytest.mark.parametrize("metric, func", PAIRED_DISTANCES.items())
-def test_paired_distances(metric, func):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_paired_distances(metric, func, csr_container):
     # Test the pairwise_distance helper function.
     rng = np.random.RandomState(0)
     # Euclidean distance should be equivalent to calling the function.
@@ -443,7 +466,7 @@ def test_paired_distances(metric, func):
     S = paired_distances(X, Y, metric=metric)
     S2 = func(X, Y)
     assert_allclose(S, S2)
-    S3 = func(csr_matrix(X), csr_matrix(Y))
+    S3 = func(csr_container(X), csr_container(Y))
     assert_allclose(S, S3)
     if metric in PAIRWISE_DISTANCE_FUNCTIONS:
         # Check the pairwise_distances implementation
@@ -473,13 +496,15 @@ def test_paired_distances_callable(global_dtype):
         paired_distances(X, Y)
 
 
-def test_pairwise_distances_argmin_min(global_dtype):
+@pytest.mark.parametrize("dok_container", DOK_CONTAINERS)
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_distances_argmin_min(dok_container, csr_container, global_dtype):
     # Check pairwise minimum distances computation for any metric
     X = np.asarray([[0], [1]], dtype=global_dtype)
     Y = np.asarray([[-2], [3]], dtype=global_dtype)
 
-    Xsp = dok_matrix(X)
-    Ysp = csr_matrix(Y, dtype=global_dtype)
+    Xsp = dok_container(X)
+    Ysp = csr_container(Y, dtype=global_dtype)
 
     expected_idx = [0, 1]
     expected_vals = [2, 2]
@@ -633,9 +658,19 @@ def test_pairwise_distances_chunked_reduce_none(global_dtype):
     [
         lambda D, start: list(D),
         lambda D, start: np.array(D),
-        lambda D, start: csr_matrix(D),
         lambda D, start: (list(D), list(D)),
-        lambda D, start: (dok_matrix(D), np.array(D), list(D)),
+    ]
+    + [
+        lambda D, start, scipy_csr_type=scipy_csr_type: scipy_csr_type(D)
+        for scipy_csr_type in CSR_CONTAINERS
+    ]
+    + [
+        lambda D, start, scipy_dok_type=scipy_dok_type: (
+            scipy_dok_type(D),
+            np.array(D),
+            list(D),
+        )
+        for scipy_dok_type in DOK_CONTAINERS
     ],
 )
 def test_pairwise_distances_chunked_reduce_valid(good_reduce):
@@ -759,10 +794,14 @@ def test_pairwise_distances_chunked(global_dtype):
 
 
 @pytest.mark.parametrize(
-    "x_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 @pytest.mark.parametrize(
-    "y_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 def test_euclidean_distances_known_result(x_array_constr, y_array_constr):
     # Check the pairwise Euclidean distances computation on known result
@@ -773,7 +812,9 @@ def test_euclidean_distances_known_result(x_array_constr, y_array_constr):
 
 
 @pytest.mark.parametrize(
-    "y_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 def test_euclidean_distances_with_norms(global_dtype, y_array_constr):
     # check that we still get the right answers with {X,Y}_norm_squared
@@ -842,10 +883,14 @@ def test_euclidean_distances_norm_shapes():
 
 
 @pytest.mark.parametrize(
-    "x_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 @pytest.mark.parametrize(
-    "y_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 def test_euclidean_distances(global_dtype, x_array_constr, y_array_constr):
     # check that euclidean distances gives same result as scipy cdist
@@ -869,7 +914,9 @@ def test_euclidean_distances(global_dtype, x_array_constr, y_array_constr):
 
 
 @pytest.mark.parametrize(
-    "x_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 def test_euclidean_distances_sym(global_dtype, x_array_constr):
     # check that euclidean distances gives same result as scipy pdist
@@ -891,10 +938,14 @@ def test_euclidean_distances_sym(global_dtype, x_array_constr):
 
 @pytest.mark.parametrize("batch_size", [None, 5, 7, 101])
 @pytest.mark.parametrize(
-    "x_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 @pytest.mark.parametrize(
-    "y_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "y_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 def test_euclidean_distances_upcast(batch_size, x_array_constr, y_array_constr):
     # check batches handling when Y != X (#13910)
@@ -918,7 +969,9 @@ def test_euclidean_distances_upcast(batch_size, x_array_constr, y_array_constr):
 
 @pytest.mark.parametrize("batch_size", [None, 5, 7, 101])
 @pytest.mark.parametrize(
-    "x_array_constr", [np.array, csr_matrix], ids=["dense", "sparse"]
+    "x_array_constr",
+    [np.array] + CSR_CONTAINERS,
+    ids=["dense"] + [container.__name__ for container in CSR_CONTAINERS],
 )
 def test_euclidean_distances_upcast_sym(batch_size, x_array_constr):
     # check batches handling when X is Y (#13910)
@@ -1267,10 +1320,11 @@ def test_kernel_symmetry(kernel):
         cosine_similarity,
     ),
 )
-def test_kernel_sparse(kernel):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_kernel_sparse(kernel, csr_container):
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
-    X_sparse = csr_matrix(X)
+    X_sparse = csr_container(X)
     K = kernel(X, X)
     K2 = kernel(X_sparse, X_sparse)
     assert_allclose(K, K2)
@@ -1305,14 +1359,16 @@ def test_laplacian_kernel():
 
 
 @pytest.mark.parametrize(
-    "metric, pairwise_func", [("linear", linear_kernel), ("cosine", cosine_similarity)]
+    "metric, pairwise_func",
+    [("linear", linear_kernel), ("cosine", cosine_similarity)],
 )
-def test_pairwise_similarity_sparse_output(metric, pairwise_func):
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_pairwise_similarity_sparse_output(metric, pairwise_func, csr_container):
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
     Y = rng.random_sample((3, 4))
-    Xcsr = csr_matrix(X)
-    Ycsr = csr_matrix(Y)
+    Xcsr = csr_container(X)
+    Ycsr = csr_container(Y)
 
     # should be sparse
     K1 = pairwise_func(Xcsr, Ycsr, dense_output=False)
@@ -1328,14 +1384,15 @@ def test_pairwise_similarity_sparse_output(metric, pairwise_func):
     assert_allclose(K1.toarray(), K3)
 
 
-def test_cosine_similarity():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_cosine_similarity(csr_container):
     # Test the cosine_similarity.
 
     rng = np.random.RandomState(0)
     X = rng.random_sample((5, 4))
     Y = rng.random_sample((3, 4))
-    Xcsr = csr_matrix(X)
-    Ycsr = csr_matrix(Y)
+    Xcsr = csr_container(X)
+    Ycsr = csr_container(Y)
 
     for X_, Y_ in ((X, None), (X, Y), (Xcsr, None), (Xcsr, Ycsr)):
         # Test that the cosine is kernel is equal to a linear kernel when data
@@ -1399,13 +1456,14 @@ def test_check_invalid_dimensions():
         check_pairwise_arrays(XA, XB)
 
 
-def test_check_sparse_arrays():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_check_sparse_arrays(csr_container):
     # Ensures that checks return valid sparse matrices.
     rng = np.random.RandomState(0)
     XA = rng.random_sample((5, 4))
-    XA_sparse = csr_matrix(XA)
+    XA_sparse = csr_container(XA)
     XB = rng.random_sample((5, 4))
-    XB_sparse = csr_matrix(XB)
+    XB_sparse = csr_container(XB)
     XA_checked, XB_checked = check_pairwise_arrays(XA_sparse, XB_sparse)
     # compare their difference because testing csr matrices for
     # equality with '==' does not work as expected.
@@ -1550,10 +1608,11 @@ def test_numeric_pairwise_distances_datatypes(metric, global_dtype, y_is_x):
     assert_allclose(dist, expected_dist)
 
 
-def test_sparse_manhattan_readonly_dataset():
+@pytest.mark.parametrize("csr_container", CSR_CONTAINERS)
+def test_sparse_manhattan_readonly_dataset(csr_container):
     # Non-regression test for: https://github.com/scikit-learn/scikit-learn/issues/7981
-    matrices1 = [csr_matrix(np.ones((5, 5)))]
-    matrices2 = [csr_matrix(np.ones((5, 5)))]
+    matrices1 = [csr_container(np.ones((5, 5)))]
+    matrices2 = [csr_container(np.ones((5, 5)))]
     # Joblib memory maps datasets which makes them read-only.
     # The following call was reporting as failing in #7981, but this must pass.
     Parallel(n_jobs=2, max_nbytes=0)(