FEA SLEP006: Metadata routing for validation_curve (scikit-learn#29329)

Co-authored-by: Adam Li <adam2392@gmail.com>

Author: StefanieSenger

doc/metadata_routing.rst

@@ -305,6 +305,7 @@ Meta-estimators and functions supporting metadata routing:
 - :func:`sklearn.model_selection.cross_val_score`
 - :func:`sklearn.model_selection.cross_val_predict`
 - :class:`sklearn.model_selection.learning_curve`
+- :class:`sklearn.model_selection.validation_curve`
 - :class:`sklearn.multiclass.OneVsOneClassifier`
 - :class:`sklearn.multiclass.OneVsRestClassifier`
 - :class:`sklearn.multiclass.OutputCodeClassifier`
@@ -323,5 +324,4 @@ Meta-estimators and tools not supporting metadata routing yet:
 - :class:`sklearn.feature_selection.RFECV`
 - :class:`sklearn.feature_selection.SequentialFeatureSelector`
 - :class:`sklearn.model_selection.permutation_test_score`
-- :class:`sklearn.model_selection.validation_curve`
 - :class:`sklearn.semi_supervised.SelfTrainingClassifier`

doc/whats_new/v1.6.rst

@@ -77,6 +77,10 @@ more details.
   params to the underlying regressor.
   :pr:`29136` by :user:`Omar Salman <OmarManzoor>`.
 
+- |Feature| :func:`model_selection.validation_curve` now supports metadata routing for
+  the `fit` method of its estimator and for its underlying CV splitter and scorer.
+  :pr:`29329` by :user:`Stefanie Senger <StefanieSenger>`.
+
 Dropping official support for PyPy
 ----------------------------------
 

sklearn/model_selection/_validation.py

@@ -1855,7 +1855,7 @@ def learning_curve(
         Parameters to pass to the fit method of the estimator.
 
         .. deprecated:: 1.6
-            This parameter is deprecated and will be removed in version 1.6. Use
+            This parameter is deprecated and will be removed in version 1.8. Use
             ``params`` instead.
 
     params : dict, default=None
@@ -2221,6 +2221,7 @@ def _incremental_fit_estimator(
         "verbose": ["verbose"],
         "error_score": [StrOptions({"raise"}), Real],
         "fit_params": [dict, None],
+        "params": [dict, None],
     },
     prefer_skip_nested_validation=False,  # estimator is not validated yet
 )
@@ -2239,6 +2240,7 @@ def validation_curve(
     verbose=0,
     error_score=np.nan,
     fit_params=None,
+    params=None,
 ):
     """Validation curve.
 
@@ -2277,6 +2279,13 @@ def validation_curve(
         train/test set. Only used in conjunction with a "Group" :term:`cv`
         instance (e.g., :class:`GroupKFold`).
 
+        .. versionchanged:: 1.6
+            ``groups`` can only be passed if metadata routing is not enabled
+            via ``sklearn.set_config(enable_metadata_routing=True)``. When routing
+            is enabled, pass ``groups`` alongside other metadata via the ``params``
+            argument instead. E.g.:
+            ``validation_curve(..., params={'groups': groups})``.
+
     cv : int, cross-validation generator or an iterable, default=None
         Determines the cross-validation splitting strategy.
         Possible inputs for cv are:
@@ -2327,7 +2336,22 @@ def validation_curve(
     fit_params : dict, default=None
         Parameters to pass to the fit method of the estimator.
 
-        .. versionadded:: 0.24
+        .. deprecated:: 1.6
+            This parameter is deprecated and will be removed in version 1.8. Use
+            ``params`` instead.
+
+    params : dict, default=None
+        Parameters to pass to the estimator, scorer and cross-validation object.
+
+            - If `enable_metadata_routing=False` (default):
+              Parameters directly passed to the `fit` method of the estimator.
+
+            - If `enable_metadata_routing=True`:
+              Parameters safely routed to the `fit` method of the estimator, to the
+              scorer and to the cross-validation object. See :ref:`Metadata Routing User
+              Guide <metadata_routing>` for more details.
+
+            .. versionadded:: 1.6
 
     Returns
     -------
@@ -2358,11 +2382,59 @@ def validation_curve(
     >>> print(f"The average test accuracy is {test_scores.mean():.2f}")
     The average test accuracy is 0.81
     """
+    params = _check_params_groups_deprecation(fit_params, params, groups, "1.8")
     X, y, groups = indexable(X, y, groups)
 
     cv = check_cv(cv, y, classifier=is_classifier(estimator))
     scorer = check_scoring(estimator, scoring=scoring)
 
+    if _routing_enabled():
+        router = (
+            MetadataRouter(owner="validation_curve")
+            .add(
+                estimator=estimator,
+                method_mapping=MethodMapping().add(caller="fit", callee="fit"),
+            )
+            .add(
+                splitter=cv,
+                method_mapping=MethodMapping().add(caller="fit", callee="split"),
+            )
+            .add(
+                scorer=scorer,
+                method_mapping=MethodMapping().add(caller="fit", callee="score"),
+            )
+        )
+
+        try:
+            routed_params = process_routing(router, "fit", **params)
+        except UnsetMetadataPassedError as e:
+            # The default exception would mention `fit` since in the above
+            # `process_routing` code, we pass `fit` as the caller. However,
+            # the user is not calling `fit` directly, so we change the message
+            # to make it more suitable for this case.
+            unrequested_params = sorted(e.unrequested_params)
+            raise UnsetMetadataPassedError(
+                message=(
+                    f"{unrequested_params} are passed to `validation_curve` but are not"
+                    " explicitly set as requested or not requested for"
+                    f" validation_curve's estimator: {estimator.__class__.__name__}."
+                    " Call `.set_fit_request({{metadata}}=True)` on the estimator for"
+                    f" each metadata in {unrequested_params} that you"
+                    " want to use and `metadata=False` for not using it. See the"
+                    " Metadata Routing User guide"
+                    " <https://scikit-learn.org/stable/metadata_routing.html> for more"
+                    " information."
+                ),
+                unrequested_params=e.unrequested_params,
+                routed_params=e.routed_params,
+            )
+
+    else:
+        routed_params = Bunch()
+        routed_params.estimator = Bunch(fit=params)
+        routed_params.splitter = Bunch(split={"groups": groups})
+        routed_params.scorer = Bunch(score={})
+
     parallel = Parallel(n_jobs=n_jobs, pre_dispatch=pre_dispatch, verbose=verbose)
     results = parallel(
         delayed(_fit_and_score)(
@@ -2374,14 +2446,13 @@ def validation_curve(
             test=test,
             verbose=verbose,
             parameters={param_name: v},
-            fit_params=fit_params,
-            # TODO(SLEP6): support score params here
-            score_params=None,
+            fit_params=routed_params.estimator.fit,
+            score_params=routed_params.scorer.score,
             return_train_score=True,
             error_score=error_score,
         )
         # NOTE do not change order of iteration to allow one time cv splitters
-        for train, test in cv.split(X, y, groups)
+        for train, test in cv.split(X, y, **routed_params.splitter.split)
         for v in param_range
     )
     n_params = len(param_range)

sklearn/model_selection/tests/test_validation.py

@@ -1697,7 +1697,7 @@ def test_validation_curve_cv_splits_consistency():
     assert_array_almost_equal(np.array(scores3), np.array(scores1))
 
 
-def test_validation_curve_fit_params():
+def test_validation_curve_params():
     X = np.arange(100).reshape(10, 10)
     y = np.array([0] * 5 + [1] * 5)
     clf = CheckingClassifier(expected_sample_weight=True)
@@ -1722,7 +1722,7 @@ def test_validation_curve_fit_params():
             param_name="foo_param",
             param_range=[1, 2, 3],
             error_score="raise",
-            fit_params={"sample_weight": np.ones(1)},
+            params={"sample_weight": np.ones(1)},
         )
     validation_curve(
         clf,
@@ -1731,7 +1731,7 @@ def test_validation_curve_fit_params():
         param_name="foo_param",
         param_range=[1, 2, 3],
         error_score="raise",
-        fit_params={"sample_weight": np.ones(10)},
+        params={"sample_weight": np.ones(10)},
     )
 
 
@@ -2482,29 +2482,54 @@ def test_cross_validate_return_indices(global_random_seed):
         assert_array_equal(test_indices[split_idx], expected_test_idx)
 
 
-# Tests for metadata routing in cross_val* and learning_curve
-# ===========================================================
+# Tests for metadata routing in cross_val* and in *curve
+# ======================================================
 
 
 # TODO(1.6): remove `cross_validate` and `cross_val_predict` from this test in 1.6 and
-# `learning_curve` in 1.8
-@pytest.mark.parametrize("func", [cross_validate, cross_val_predict, learning_curve])
-def test_fit_param_deprecation(func):
+# `learning_curve` and `validation_curve` in 1.8
+@pytest.mark.parametrize(
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
+)
+def test_fit_param_deprecation(func, extra_args):
     """Check that we warn about deprecating `fit_params`."""
     with pytest.warns(FutureWarning, match="`fit_params` is deprecated"):
-        func(estimator=ConsumingClassifier(), X=X, y=y, cv=2, fit_params={})
+        func(
+            estimator=ConsumingClassifier(), X=X, y=y, cv=2, fit_params={}, **extra_args
+        )
 
     with pytest.raises(
         ValueError, match="`params` and `fit_params` cannot both be provided"
     ):
-        func(estimator=ConsumingClassifier(), X=X, y=y, fit_params={}, params={})
+        func(
+            estimator=ConsumingClassifier(),
+            X=X,
+            y=y,
+            fit_params={},
+            params={},
+            **extra_args,
+        )
 
 
 @pytest.mark.usefixtures("enable_slep006")
 @pytest.mark.parametrize(
-    "func", [cross_validate, cross_val_score, cross_val_predict, learning_curve]
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
 )
-def test_groups_with_routing_validation(func):
+def test_groups_with_routing_validation(func, extra_args):
     """Check that we raise an error if `groups` are passed to the cv method instead
     of `params` when metadata routing is enabled.
     """
@@ -2514,14 +2539,22 @@ def test_groups_with_routing_validation(func):
             X=X,
             y=y,
             groups=[],
+            **extra_args,
         )
 
 
 @pytest.mark.usefixtures("enable_slep006")
 @pytest.mark.parametrize(
-    "func", [cross_validate, cross_val_score, cross_val_predict, learning_curve]
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
 )
-def test_passed_unrequested_metadata(func):
+def test_passed_unrequested_metadata(func, extra_args):
     """Check that we raise an error when passing metadata that is not
     requested."""
     err_msg = re.escape("but are not explicitly set as requested or not requested")
@@ -2531,14 +2564,22 @@ def test_passed_unrequested_metadata(func):
             X=X,
             y=y,
             params=dict(metadata=[]),
+            **extra_args,
         )
 
 
 @pytest.mark.usefixtures("enable_slep006")
 @pytest.mark.parametrize(
-    "func", [cross_validate, cross_val_score, cross_val_predict, learning_curve]
+    "func, extra_args",
+    [
+        (cross_validate, {}),
+        (cross_val_score, {}),
+        (cross_val_predict, {}),
+        (learning_curve, {}),
+        (validation_curve, {"param_name": "alpha", "param_range": np.array([1])}),
+    ],
 )
-def test_validation_functions_routing(func):
+def test_validation_functions_routing(func, extra_args):
     """Check that the respective cv method is properly dispatching the metadata
     to the consumer."""
     scorer_registry = _Registry()
@@ -2563,12 +2604,11 @@ def test_validation_functions_routing(func):
     fit_sample_weight = rng.rand(n_samples)
     fit_metadata = rng.rand(n_samples)
 
-    extra_params = {
+    scoring_args = {
         cross_validate: dict(scoring=dict(my_scorer=scorer, accuracy="accuracy")),
-        # cross_val_score and learning_curve don't support multiple scorers:
         cross_val_score: dict(scoring=scorer),
         learning_curve: dict(scoring=scorer),
-        # cross_val_predict doesn't need a scorer
+        validation_curve: dict(scoring=scorer),
         cross_val_predict: dict(),
     }
 
@@ -2590,7 +2630,8 @@ def test_validation_functions_routing(func):
         X=X,
         y=y,
         cv=splitter,
-        **extra_params[func],
+        **scoring_args[func],
+        **extra_args,
         params=params,
     )