Updated Auto-Select Reporting in Forecast Operator (#1013)

Author: prasankh

ads/opctl/operator/lowcode/forecast/const.py

@@ -87,3 +87,4 @@ class ForecastOutputColumns(str, metaclass=ExtendedEnumMeta):
 PROPHET_INTERNAL_DATE_COL = "ds"
 RENDER_LIMIT = 5000
 AUTO_SELECT = "auto-select"
+BACKTEST_REPORT_NAME = "back_test.csv"

ads/opctl/operator/lowcode/forecast/model/base_model.py

@@ -47,6 +47,7 @@
     SpeedAccuracyMode,
     SupportedMetrics,
     SupportedModels,
+    BACKTEST_REPORT_NAME
 )
 from ..operator_config import ForecastOperatorConfig, ForecastOperatorSpec
 from .forecast_datasets import ForecastDatasets
@@ -256,12 +257,9 @@ def generate_report(self):
 
                 backtest_sections = []
                 output_dir = self.spec.output_directory.url
-                backtest_report_name = "backtest_stats.csv"
-                file_path = f"{output_dir}/{backtest_report_name}"
+                file_path = f"{output_dir}/{BACKTEST_REPORT_NAME}"
                 if self.spec.model == AUTO_SELECT:
-                    backtest_sections.append(
-                        rc.Heading("Auto-select statistics", level=2)
-                    )
+                    backtest_sections.append(rc.Heading("Auto-Select Backtesting and Performance Metrics", level=2))
                     if not os.path.exists(file_path):
                         failure_msg = rc.Text(
                             "auto-select could not be executed. Please check the "
@@ -270,19 +268,15 @@ def generate_report(self):
                         backtest_sections.append(failure_msg)
                     else:
                         backtest_stats = pd.read_csv(file_path)
-                        average_dict = backtest_stats.mean().to_dict()
-                        del average_dict["backtest"]
+                        model_metric_map = backtest_stats.drop(columns=['metric', 'backtest'])
+                        average_dict = {k: round(v, 4) for k, v in model_metric_map.mean().to_dict().items()}
                         best_model = min(average_dict, key=average_dict.get)
-                        backtest_text = rc.Heading("Back Testing Metrics", level=3)
                         summary_text = rc.Text(
-                            f"Overall, the average scores for the models are {average_dict}, with {best_model}"
-                            f" being identified as the top-performing model during backtesting."
-                        )
+                            f"Overall, the average {self.spec.metric} scores for the models are {average_dict}, with"
+                            f" {best_model} being identified as the top-performing model during backtesting.")
                         backtest_table = rc.DataTable(backtest_stats, index=True)
                         liner_plot = get_auto_select_plot(backtest_stats)
-                        backtest_sections.extend(
-                            [backtest_text, backtest_table, summary_text, liner_plot]
-                        )
+                        backtest_sections.extend([backtest_table, summary_text, liner_plot])
 
                 forecast_plots = []
                 if len(self.forecast_output.list_series_ids()) > 0:

ads/opctl/operator/lowcode/forecast/model_evaluator.py

@@ -10,6 +10,7 @@
 
 from ads.opctl import logger
 from ads.opctl.operator.lowcode.common.const import DataColumns
+from ads.opctl.operator.lowcode.forecast.const import BACKTEST_REPORT_NAME
 from .model.forecast_datasets import ForecastDatasets
 from .operator_config import ForecastOperatorConfig
 from ads.opctl.operator.lowcode.forecast.model.factory import SupportedModels
@@ -156,8 +157,8 @@ def find_best_model(self, datasets: ForecastDatasets, operator_config: ForecastO
         best_model = min(avg_backtests_metric, key=avg_backtests_metric.get)
         logger.info(f"Among models {self.models}, {best_model} model shows better performance during backtesting.")
         backtest_stats = pd.DataFrame(nonempty_metrics).rename_axis('backtest')
+        backtest_stats["metric"] = operator_config.spec.metric
         backtest_stats.reset_index(inplace=True)
         output_dir = operator_config.spec.output_directory.url
-        backtest_report_name = "backtest_stats.csv"
-        backtest_stats.to_csv(f"{output_dir}/{backtest_report_name}", index=False)
+        backtest_stats.to_csv(f"{output_dir}/{BACKTEST_REPORT_NAME}", index=False)
         return best_model

ads/opctl/operator/lowcode/forecast/utils.py

@@ -261,10 +261,11 @@ def _add_unit(num, unit):
 
 def get_auto_select_plot(backtest_results):
     fig = go.Figure()
-    columns = backtest_results.columns.tolist()
+    back_test_csv_columns = backtest_results.columns.tolist()
     back_test_column = "backtest"
-    columns.remove(back_test_column)
-    for column in columns:
+    metric_column = "metric"
+    models = [x for x in back_test_csv_columns if x not in [back_test_column, metric_column]]
+    for i, column in enumerate(models):
         fig.add_trace(
             go.Scatter(
                 x=backtest_results[back_test_column],