[ENH] Forecasting testing (#2891)

* call forecasting checks

* basic cleanup

* fix

* add assert error messages

* remove fit_is_empty check (see #2893)

* remove extra argument to fit

* remove extra argument to predict

* correct forecast argument

* Update _yield_forecasting_checks.py

* add test for predict

---------

Co-authored-by: Tony Bagnall <ajb@uea.ac.uk>

Author: MatthewMiddlehurst

aeon/testing/estimator_checking/_yield_anomaly_detection_checks.py

@@ -38,7 +38,7 @@ def check_anomaly_detector_overrides_and_tags(estimator_class):
 
     # Test that fit_is_empty is correctly set
     fit_is_empty = estimator_class.get_class_tag(tag_name="fit_is_empty")
-    assert not fit_is_empty == "_fit" not in estimator_class.__dict__
+    assert fit_is_empty == ("_fit" not in estimator_class.__dict__)
 
     # Test valid tag for X_inner_type
     X_inner_type = estimator_class.get_class_tag(tag_name="X_inner_type")

aeon/testing/estimator_checking/_yield_estimator_checks.py

@@ -21,6 +21,7 @@
 from aeon.classification.early_classification import BaseEarlyClassifier
 from aeon.clustering import BaseClusterer
 from aeon.clustering.deep_learning.base import BaseDeepClusterer
+from aeon.forecasting import BaseForecaster
 from aeon.regression import BaseRegressor
 from aeon.regression.deep_learning.base import BaseDeepRegressor
 from aeon.segmentation import BaseSegmenter
@@ -40,6 +41,9 @@
 from aeon.testing.estimator_checking._yield_early_classification_checks import (
     _yield_early_classification_checks,
 )
+from aeon.testing.estimator_checking._yield_forecasting_checks import (
+    _yield_forecasting_checks,
+)
 from aeon.testing.estimator_checking._yield_multithreading_checks import (
     _yield_multithreading_checks,
 )
@@ -158,6 +162,11 @@ def _yield_all_aeon_checks(
             estimator_class, estimator_instances, datatypes
         )
 
+    if issubclass(estimator_class, BaseForecaster):
+        yield from _yield_forecasting_checks(
+            estimator_class, estimator_instances, datatypes
+        )
+
     if issubclass(estimator_class, BaseTransformer):
         yield from _yield_transformation_checks(
             estimator_class, estimator_instances, datatypes

aeon/testing/estimator_checking/_yield_forecasting_checks.py

@@ -2,50 +2,77 @@
 
 from functools import partial
 
-import numpy as np
-
 from aeon.base._base import _clone_estimator
-from aeon.utils.data_types import VALID_SERIES_INPUT_TYPES
+from aeon.testing.testing_data import FULL_TEST_DATA_DICT
+from aeon.utils.data_types import VALID_SERIES_INNER_TYPES
 
 
 def _yield_forecasting_checks(estimator_class, estimator_instances, datatypes):
     """Yield all forecasting checks for an aeon forecaster."""
     # only class required
-    yield partial(check_forecasting_base_functionality, estimator_class=estimator_class)
+    yield partial(check_forecaster_overrides_and_tags, estimator_class=estimator_class)
 
     # test class instances
-    for _, estimator in enumerate(estimator_instances):
-        # no data needed
-        yield partial(check_forecaster_instance, estimator=estimator)
-
-
-def check_forecasting_base_functionality(estimator_class):
-    """Test compliance with the base class contract."""
-    # Test they dont override final methods, because python does not enforce this
-    assert "fit" not in estimator_class.__dict__
-    assert "predict" not in estimator_class.__dict__
-    assert "forecast" not in estimator_class.__dict__
-    fit_is_empty = estimator_class.get_class_tag(tag_name="fit_is_empty")
-    assert not fit_is_empty == "_fit" not in estimator_class.__dict__
+    for i, estimator in enumerate(estimator_instances):
+        # test all data types
+        for datatype in datatypes[i]:
+            yield partial(
+                check_forecaster_output,
+                estimator=estimator,
+                datatype=datatype,
+            )
+
+
+def check_forecaster_overrides_and_tags(estimator_class):
+    """Test compliance with the forecaster base class contract."""
+    # Test they don't override final methods, because Python does not enforce this
+    final_methods = ["fit", "predict", "forecast"]
+    for method in final_methods:
+        if method in estimator_class.__dict__:
+            raise ValueError(
+                f"Forecaster {estimator_class} overrides the "
+                f"method {method}. Override _{method} instead."
+            )
+
+    # Test that all forecasters implement abstract predict.
+    assert "_predict" in estimator_class.__dict__
+
+    # todo decide what to do with "fit_is_empty" and abstract "_fit"
+
     # Test valid tag for X_inner_type
     X_inner_type = estimator_class.get_class_tag(tag_name="X_inner_type")
-    assert X_inner_type in VALID_SERIES_INPUT_TYPES
+    if isinstance(X_inner_type, str):
+        assert X_inner_type in VALID_SERIES_INNER_TYPES
+    else:  # must be a list
+        assert all([t in VALID_SERIES_INNER_TYPES for t in X_inner_type])
+
     # Must have at least one set to True
     multi = estimator_class.get_class_tag(tag_name="capability:multivariate")
     uni = estimator_class.get_class_tag(tag_name="capability:univariate")
-    assert multi or uni
+    assert multi or uni, (
+        "At least one of tag capability:multivariate or "
+        "capability:univariate must be true."
+    )
 
 
-def check_forecaster_instance(estimator):
-    """Test forecasters."""
+def check_forecaster_output(estimator, datatype):
+    """Test the forecaster output on valid data."""
     estimator = _clone_estimator(estimator)
-    pass
-    # Sort
-    # Check output correct: predict should return a float
-    y = np.array([0.5, 0.7, 0.8, 0.9, 1.0])
-    estimator.fit(y)
-    p = estimator.predict()
-    assert isinstance(p, float)
-    # forecast should return a float equal to fit/predict
-    p2 = estimator.forecast(y)
-    assert p == p2
+
+    estimator.fit(
+        FULL_TEST_DATA_DICT[datatype]["train"][0],
+    )
+    y_pred = estimator.predict()
+    assert isinstance(y_pred, float), (
+        f"predict() output should be float, got" f" {type(y_pred)}"
+    )
+
+    y_pred2 = estimator.forecast(FULL_TEST_DATA_DICT[datatype]["train"][0])
+    assert y_pred == y_pred2, (
+        f"predict() and forecast() output differ: {y_pred} !=" f" {y_pred2}"
+    )
+    y_pred3 = estimator.predict(FULL_TEST_DATA_DICT[datatype]["train"][0])
+    assert y_pred == y_pred3, (
+        f"after fit(), predict() and predict(y_train) should be the same, but"
+        f"output differ: {y_pred} != {y_pred3}"
+    )