FEAT SLEP006 permutation_test_score to support metadata routing (scikit-learn#29266)

Author: adam2392

doc/metadata_routing.rst

@@ -301,6 +301,7 @@ Meta-estimators and functions supporting metadata routing:
 - :class:`sklearn.model_selection.HalvingGridSearchCV`
 - :class:`sklearn.model_selection.HalvingRandomSearchCV`
 - :class:`sklearn.model_selection.RandomizedSearchCV`
+- :class:`sklearn.model_selection.permutation_test_score`
 - :func:`sklearn.model_selection.cross_validate`
 - :func:`sklearn.model_selection.cross_val_score`
 - :func:`sklearn.model_selection.cross_val_predict`
@@ -324,4 +325,3 @@ Meta-estimators and tools not supporting metadata routing yet:
 - :class:`sklearn.feature_selection.RFE`
 - :class:`sklearn.feature_selection.RFECV`
 - :class:`sklearn.feature_selection.SequentialFeatureSelector`
-- :class:`sklearn.model_selection.permutation_test_score`

doc/whats_new/v1.6.rst

@@ -89,6 +89,10 @@ more details.
   passed to the underlying estimators via their respective methods.
   :pr:`28494` by :user:`Adam Li <adam2392>`.
 
+- |Feature| :func:`model_selection.permutation_test_score` now supports metadata routing
+  for the `fit` method of its estimator and for its underlying CV splitter and scorer.
+  :pr:`29266` by :user:`Adam Li <adam2392>`.
+
 Dropping support for building with setuptools
 ---------------------------------------------
 

sklearn/model_selection/_validation.py

@@ -1493,6 +1493,7 @@ def _check_is_permutation(indices, n_samples):
         "verbose": ["verbose"],
         "scoring": [StrOptions(set(get_scorer_names())), callable, None],
         "fit_params": [dict, None],
+        "params": [dict, None],
     },
     prefer_skip_nested_validation=False,  # estimator is not validated yet
 )
@@ -1509,6 +1510,7 @@ def permutation_test_score(
     verbose=0,
     scoring=None,
     fit_params=None,
+    params=None,
 ):
     """Evaluate the significance of a cross-validated score with permutations.
 
@@ -1548,6 +1550,13 @@ def permutation_test_score(
         cross-validator uses them for grouping the samples  while splitting
         the dataset into train/test set.
 
+        .. versionchanged:: 1.6
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``permutation_test_score(..., params={'groups': groups})``.
+
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
@@ -1594,7 +1603,24 @@ def permutation_test_score(
     fit_params : dict, default=None
         Parameters to pass to the fit method of the estimator.
 
-        .. versionadded:: 0.24
+        .. deprecated:: 1.6
+            This parameter is deprecated and will be removed in version 1.6. Use
+            ``params`` instead.
+
+    params : dict, default=None
+        Parameters to pass to the `fit` method of the estimator, the scorer
+        and the cv splitter.
+
+            - If `enable_metadata_routing=False` (default):
+              Parameters directly passed to the `fit` method of the estimator.
+
+            - If `enable_metadata_routing=True`:
+              Parameters safely routed to the `fit` method of the estimator,
+              `cv` object and `scorer`.
+              See :ref:`Metadata Routing User Guide <metadata_routing>` for more
+              details.
+
+            .. versionadded:: 1.6
 
     Returns
     -------
@@ -1643,26 +1669,86 @@ def permutation_test_score(
     >>> print(f"P-value: {pvalue:.3f}")
     P-value: 0.010
     """
+    params = _check_params_groups_deprecation(fit_params, params, groups, "1.8")
+
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
     scorer = check_scoring(estimator, scoring=scoring)
     random_state = check_random_state(random_state)
 
+    if _routing_enabled():
+        router = (
+            MetadataRouter(owner="permutation_test_score")
+            .add(
+                estimator=estimator,
+                # TODO(SLEP6): also pass metadata to the predict method for
+                # scoring?
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                scorer=scorer,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+        )
+
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            unrequested_params = sorted(e.unrequested_params)
+            raise UnsetMetadataPassedError(
+                message=(
+                    f"{unrequested_params} are passed to `permutation_test_score`"
+                    " but are not explicitly set as requested or not requested"
+                    " for permutation_test_score's"
+                    f" estimator: {estimator.__class__.__name__}. Call"
+                    " `.set_fit_request({{metadata}}=True)` on the estimator for"
+                    f" each metadata in {unrequested_params} that you"
+                    " want to use and `metadata=False` for not using it. See the"
+                    " Metadata Routing User guide"
+                    " <https://scikit-learn.org/stable/metadata_routing.html> for more"
+                    " information."
+                ),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
+
+    else:
+        routed_params = Bunch()
+        routed_params.estimator = Bunch(fit=params)
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.scorer = Bunch(score={})
+
     # We clone the estimator to make sure that all the folds are
     # independent, and that it is pickle-able.
     score = _permutation_test_score(
-        clone(estimator), X, y, groups, cv, scorer, fit_params=fit_params
+        clone(estimator),
+        X,
+        y,
+        cv,
+        scorer,
+        split_params=routed_params.splitter.split,
+        fit_params=routed_params.estimator.fit,
+        score_params=routed_params.scorer.score,
     )
     permutation_scores = Parallel(n_jobs=n_jobs, verbose=verbose)(
         delayed(_permutation_test_score)(
             clone(estimator),
             X,
             _shuffle(y, groups, random_state),
-            groups,
             cv,
             scorer,
-            fit_params=fit_params,
+            split_params=routed_params.splitter.split,
+            fit_params=routed_params.estimator.fit,
+            score_params=routed_params.scorer.score,
         )
         for _ in range(n_permutations)
     )
@@ -1671,17 +1757,22 @@ def permutation_test_score(
     return score, permutation_scores, pvalue
 
 
-def _permutation_test_score(estimator, X, y, groups, cv, scorer, fit_params):
+def _permutation_test_score(
+    estimator, X, y, cv, scorer, split_params, fit_params, score_params
+):
     """Auxiliary function for permutation_test_score"""
     # Adjust length of sample weights
     fit_params = fit_params if fit_params is not None else {}
+    score_params = score_params if score_params is not None else {}
+
     avg_score = []
-    for train, test in cv.split(X, y, groups):
+    for train, test in cv.split(X, y, **split_params):
         X_train, y_train = _safe_split(estimator, X, y, train)
         X_test, y_test = _safe_split(estimator, X, y, test, train)
-        fit_params = _check_method_params(X, params=fit_params, indices=train)
-        estimator.fit(X_train, y_train, **fit_params)
-        avg_score.append(scorer(estimator, X_test, y_test))
+        fit_params_train = _check_method_params(X, params=fit_params, indices=train)
+        score_params_test = _check_method_params(X, params=score_params, indices=test)
+        estimator.fit(X_train, y_train, **fit_params_train)
+        avg_score.append(scorer(estimator, X_test, y_test, **score_params_test))
     return np.mean(avg_score)
 
 

sklearn/model_selection/tests/test_validation.py

@@ -862,7 +862,7 @@ def test_permutation_test_score_allow_nans():
     permutation_test_score(p, X, y)
 
 
-def test_permutation_test_score_fit_params():
+def test_permutation_test_score_params():
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
     clf = CheckingClassifier(expected_sample_weight=True)
@@ -873,8 +873,8 @@ def test_permutation_test_score_fit_params():
 
     err_msg = r"sample_weight.shape == \(1,\), expected \(8,\)!"
     with pytest.raises(ValueError, match=err_msg):
-        permutation_test_score(clf, X, y, fit_params={"sample_weight": np.ones(1)})
-    permutation_test_score(clf, X, y, fit_params={"sample_weight": np.ones(10)})
+        permutation_test_score(clf, X, y, params={"sample_weight": np.ones(1)})
+    permutation_test_score(clf, X, y, params={"sample_weight": np.ones(10)})
 
 
 def test_cross_val_score_allow_nans():
@@ -2495,6 +2495,7 @@ def test_cross_validate_return_indices(global_random_seed):
         (cross_val_score, {}),
         (cross_val_predict, {}),
         (learning_curve, {}),
+        (permutation_test_score, {}),
         (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
     ],
 )
@@ -2526,6 +2527,7 @@ def test_fit_param_deprecation(func, extra_args):
         (cross_val_score, {}),
         (cross_val_predict, {}),
         (learning_curve, {}),
+        (permutation_test_score, {}),
         (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
     ],
 )
@@ -2551,6 +2553,7 @@ def test_groups_with_routing_validation(func, extra_args):
         (cross_val_score, {}),
         (cross_val_predict, {}),
         (learning_curve, {}),
+        (permutation_test_score, {}),
         (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
     ],
 )
@@ -2576,6 +2579,7 @@ def test_passed_unrequested_metadata(func, extra_args):
         (cross_val_score, {}),
         (cross_val_predict, {}),
         (learning_curve, {}),
+        (permutation_test_score, {}),
         (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
     ],
 )
@@ -2609,6 +2613,7 @@ def test_validation_functions_routing(func, extra_args):
         cross_val_score: dict(scoring=scorer),
         learning_curve: dict(scoring=scorer),
         validation_curve: dict(scoring=scorer),
+        permutation_test_score: dict(scoring=scorer),
         cross_val_predict: dict(),
     }