MNT remove default behaviour deprecation from class_likelihood_ratios (scikit-learn#31331)

Co-authored-by: Jérémie du Boisberranger <jeremie@probabl.ai>

Author: StefanieSenger

doc/whats_new/upcoming_changes/sklearn.metrics/29288.api.rst

@@ -0,0 +1,4 @@
+- The `raise_warning` parameter of :func:`metrics.class_likelihood_ratios` is deprecated
+  and will be removed in 1.9. An `UndefinedMetricWarning` will always be raised in case
+  of a division by zero.
+  By :user:`Stefanie Senger <StefanieSenger>`.

sklearn/metrics/_classification.py

@@ -2052,7 +2052,6 @@ def precision_recall_fscore_support(
         "sample_weight": ["array-like", None],
         "raise_warning": ["boolean", Hidden(StrOptions({"deprecated"}))],
         "replace_undefined_by": [
-            Hidden(StrOptions({"default"})),
             Options(Real, {1.0, np.nan}),
             dict,
         ],
@@ -2066,7 +2065,7 @@ def class_likelihood_ratios(
     labels=None,
     sample_weight=None,
     raise_warning="deprecated",
-    replace_undefined_by="default",
+    replace_undefined_by=np.nan,
 ):
     """Compute binary classification positive and negative likelihood ratios.
 
@@ -2178,35 +2177,29 @@ class are present in `y_true`): both likelihood ratios are undefined.
     --------
     >>> import numpy as np
     >>> from sklearn.metrics import class_likelihood_ratios
-    >>> class_likelihood_ratios([0, 1, 0, 1, 0], [1, 1, 0, 0, 0],
-    ...                          replace_undefined_by=1.0)
+    >>> class_likelihood_ratios([0, 1, 0, 1, 0], [1, 1, 0, 0, 0])
     (1.5, 0.75)
     >>> y_true = np.array(["non-cat", "cat", "non-cat", "cat", "non-cat"])
     >>> y_pred = np.array(["cat", "cat", "non-cat", "non-cat", "non-cat"])
-    >>> class_likelihood_ratios(y_true, y_pred, replace_undefined_by=1.0)
+    >>> class_likelihood_ratios(y_true, y_pred)
     (1.33, 0.66)
     >>> y_true = np.array(["non-zebra", "zebra", "non-zebra", "zebra", "non-zebra"])
     >>> y_pred = np.array(["zebra", "zebra", "non-zebra", "non-zebra", "non-zebra"])
-    >>> class_likelihood_ratios(y_true, y_pred, replace_undefined_by=1.0)
+    >>> class_likelihood_ratios(y_true, y_pred)
     (1.5, 0.75)
 
     To avoid ambiguities, use the notation `labels=[negative_class,
     positive_class]`
 
     >>> y_true = np.array(["non-cat", "cat", "non-cat", "cat", "non-cat"])
     >>> y_pred = np.array(["cat", "cat", "non-cat", "non-cat", "non-cat"])
-    >>> class_likelihood_ratios(y_true, y_pred, labels=["non-cat", "cat"],
-    ...                          replace_undefined_by=1.0)
+    >>> class_likelihood_ratios(y_true, y_pred, labels=["non-cat", "cat"])
     (1.5, 0.75)
     """
     # TODO(1.9): When `raise_warning` is removed, the following changes need to be made:
     # The checks for `raise_warning==True` need to be removed and we will always warn,
-    # the default return value of `replace_undefined_by` should be updated from `np.nan`
-    # (which was kept for backwards compatibility) to `1.0`, its hidden option
-    # ("default") is not used anymore, some warning messages can be removed, the Warns
-    # section in the docstring should not mention `raise_warning` anymore and the
-    # "Mathematical divergences" section in model_evaluation.rst needs to be updated on
-    # the new default behaviour of `replace_undefined_by`.
+    # remove `FutureWarning`, and the Warns section in the docstring should not mention
+    # `raise_warning` anymore.
     y_true, y_pred = attach_unique(y_true, y_pred)
     y_type, y_true, y_pred = _check_targets(y_true, y_pred)
     if y_type != "binary":
@@ -2220,28 +2213,11 @@ class are present in `y_true`): both likelihood ratios are undefined.
         "`UndefinedMetricWarning` will always be raised in case of a division by zero "
         "and the value set with the `replace_undefined_by` param will be returned."
     )
-    mgs_changed_default = (
-        "The default return value of `class_likelihood_ratios` in case of a division "
-        "by zero has been deprecated in 1.7 and will be changed to the worst scores "
-        "(`(1.0, 1.0)`) in version 1.9. Set `replace_undefined_by=1.0` to use the new"
-        "default and to silence this Warning."
-    )
     if raise_warning != "deprecated":
-        warnings.warn(
-            " ".join((msg_deprecated_param, mgs_changed_default)), FutureWarning
-        )
+        warnings.warn(msg_deprecated_param, FutureWarning)
     else:
-        if replace_undefined_by == "default":
-            # TODO(1.9): Remove. If users don't set any return values in case of a
-            # division by zero (`raise_warning="deprecated"` and
-            # `replace_undefined_by="default"`) they still get a FutureWarning about
-            # changing default return values:
-            warnings.warn(mgs_changed_default, FutureWarning)
         raise_warning = True
 
-    if replace_undefined_by == "default":
-        replace_undefined_by = np.nan
-
     if replace_undefined_by == 1.0:
         replace_undefined_by = {"LR+": 1.0, "LR-": 1.0}
 
@@ -2293,12 +2269,12 @@ class are present in `y_true`): both likelihood ratios are undefined.
 
     # if `support_pos == 0`a division by zero will occur
     if support_pos == 0:
-        # TODO(1.9): Change return values in warning message to new default: the worst
-        # possible scores: `(1.0, 1.0)`
         msg = (
             "No samples of the positive class are present in `y_true`. "
             "`positive_likelihood_ratio` and `negative_likelihood_ratio` are both set "
-            "to `np.nan`."
+            "to `np.nan`. Use the `replace_undefined_by` param to control this "
+            "behavior. To suppress this warning or turn it into an error, see Python's "
+            "`warnings` module and `warnings.catch_warnings()`."
         )
         warnings.warn(msg, UndefinedMetricWarning, stacklevel=2)
         positive_likelihood_ratio = np.nan
@@ -2315,9 +2291,8 @@ class are present in `y_true`): both likelihood ratios are undefined.
             else:
                 msg_beginning = "`positive_likelihood_ratio` is ill-defined and "
             msg_end = "set to `np.nan`. Use the `replace_undefined_by` param to "
-            "control this behavior."
-            # TODO(1.9): Change return value in warning message to new default: `1.0`,
-            # which is the worst possible score for "LR+"
+            "control this behavior. To suppress this warning or turn it into an error, "
+            "see Python's `warnings` module and `warnings.catch_warnings()`."
             warnings.warn(msg_beginning + msg_end, UndefinedMetricWarning, stacklevel=2)
         if isinstance(replace_undefined_by, float) and np.isnan(replace_undefined_by):
             positive_likelihood_ratio = replace_undefined_by
@@ -2332,11 +2307,11 @@ class are present in `y_true`): both likelihood ratios are undefined.
     # if `tn == 0`a division by zero will occur
     if tn == 0:
         if raise_warning:
-            # TODO(1.9): Change return value in warning message to new default: `1.0`,
-            # which is the worst possible score for "LR-"
             msg = (
                 "`negative_likelihood_ratio` is ill-defined and set to `np.nan`. "
-                "Use the `replace_undefined_by` param to control this behavior."
+                "Use the `replace_undefined_by` param to control this behavior. To "
+                "suppress this warning or turn it into an error, see Python's "
+                "`warnings` module and `warnings.catch_warnings()`."
             )
             warnings.warn(msg, UndefinedMetricWarning, stacklevel=2)
         if isinstance(replace_undefined_by, float) and np.isnan(replace_undefined_by):

sklearn/metrics/tests/test_classification.py

@@ -709,9 +709,7 @@ def test_likelihood_ratios_warnings(params, warn_msg):
     # least one of the ratios is ill-defined.
 
     with pytest.warns(UserWarning, match=warn_msg):
-        # TODO(1.9): remove setting `replace_undefined_by` since this will be set by
-        # default
-        class_likelihood_ratios(replace_undefined_by=1.0, **params)
+        class_likelihood_ratios(**params)
 
 
 @pytest.mark.parametrize(
@@ -736,32 +734,27 @@ def test_likelihood_ratios_errors(params, err_msg):
         class_likelihood_ratios(**params)
 
 
-# TODO(1.9): remove setting `replace_undefined_by` since this will be set by default
 def test_likelihood_ratios():
     # Build confusion matrix with tn=9, fp=8, fn=1, tp=2,
     # sensitivity=2/3, specificity=9/17, prevalence=3/20,
     # LR+=34/24, LR-=17/27
     y_true = np.array([1] * 3 + [0] * 17)
     y_pred = np.array([1] * 2 + [0] * 10 + [1] * 8)
 
-    pos, neg = class_likelihood_ratios(y_true, y_pred, replace_undefined_by=np.nan)
+    pos, neg = class_likelihood_ratios(y_true, y_pred)
     assert_allclose(pos, 34 / 24)
     assert_allclose(neg, 17 / 27)
 
     # Build limit case with y_pred = y_true
-    pos, neg = class_likelihood_ratios(y_true, y_true, replace_undefined_by=np.nan)
-    # TODO(1.9): replace next line with `assert_array_equal(pos, 1.0)`, since
-    # `replace_undefined_by` has a new default:
+    pos, neg = class_likelihood_ratios(y_true, y_true)
     assert_array_equal(pos, np.nan * 2)
     assert_allclose(neg, np.zeros(2), rtol=1e-12)
 
     # Ignore last 5 samples to get tn=9, fp=3, fn=1, tp=2,
     # sensitivity=2/3, specificity=9/12, prevalence=3/20,
     # LR+=24/9, LR-=12/27
     sample_weight = np.array([1.0] * 15 + [0.0] * 5)
-    pos, neg = class_likelihood_ratios(
-        y_true, y_pred, sample_weight=sample_weight, replace_undefined_by=np.nan
-    )
+    pos, neg = class_likelihood_ratios(y_true, y_pred, sample_weight=sample_weight)
     assert_allclose(pos, 24 / 9)
     assert_allclose(neg, 12 / 27)
 
@@ -779,18 +772,6 @@ def test_likelihood_ratios_raise_warning_deprecation(raise_warning):
         class_likelihood_ratios(y_true, y_pred, raise_warning=raise_warning)
 
 
-# TODO(1.9): remove test
-def test_likelihood_ratios_raise_default_deprecation():
-    """Test that class_likelihood_ratios raises a `FutureWarning` when `raise_warning`
-    and `replace_undefined_by` are both default."""
-    y_true = np.array([1, 0])
-    y_pred = np.array([1, 0])
-
-    msg = "The default return value of `class_likelihood_ratios` in case of a"
-    with pytest.warns(FutureWarning, match=msg):
-        class_likelihood_ratios(y_true, y_pred)
-
-
 def test_likelihood_ratios_replace_undefined_by_worst():
     """Test that class_likelihood_ratios returns the worst scores `1.0` for both LR+ and
     LR- when `replace_undefined_by=1` is set."""