logging error in case of all forecast is NaN

prasankh · prasankh · commit a5698d38a6d8 · 2024-08-20T17:00:26.000+05:30
diff --git a/ads/opctl/operator/lowcode/forecast/model/arima.py b/ads/opctl/operator/lowcode/forecast/model/arima.py
@@ -127,7 +127,8 @@ def _train_model(self, i, s_id, df, model_kwargs):
             logger.debug("===========Done===========")
         except Exception as e:
             self.errors_dict[s_id] = {"model_name": self.spec.model, "error": str(e)}
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
 
     def _build_model(self) -> pd.DataFrame:
         full_data_dict = self.datasets.get_data_by_series()
diff --git a/ads/opctl/operator/lowcode/forecast/model/automlx.py b/ads/opctl/operator/lowcode/forecast/model/automlx.py
@@ -149,6 +149,9 @@ def _build_model(self) -> pd.DataFrame:
                 if f"{target}_ci_lower" not in summary_frame:
                     summary_frame[f"{target}_ci_lower"] = np.NAN
 
+                if summary_frame[target].isna().all():
+                    raise ValueError("The forecasts are completely NaN")
+
                 self.forecast_output.populate_series_output(
                     series_id=s_id,
                     fit_val=fitted_values,
@@ -168,7 +171,8 @@ def _build_model(self) -> pd.DataFrame:
                     "model_name": self.spec.model,
                     "error": str(e),
                 }
-                logger.debug(f"Encountered Error: {e}. Skipping.")
+                logger.warn(f"Encountered Error: {e}. Skipping.")
+                logger.warn(traceback.format_exc())
 
         logger.debug("===========Forecast Generated===========")
 
diff --git a/ads/opctl/operator/lowcode/forecast/model/autots.py b/ads/opctl/operator/lowcode/forecast/model/autots.py
@@ -209,7 +209,8 @@ def _build_model(self) -> pd.DataFrame:
                     "model_name": self.spec.model,
                     "error": str(e),
                 }
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
 
         logger.debug("===========Done===========")
 
diff --git a/ads/opctl/operator/lowcode/forecast/model/ml_forecast.py b/ads/opctl/operator/lowcode/forecast/model/ml_forecast.py
@@ -165,7 +165,8 @@ def _train_model(self, data_train, data_test, model_kwargs):
                 "model_name": self.spec.model,
                 "error": str(e),
             }
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
             raise e
 
     def _build_model(self) -> pd.DataFrame:
diff --git a/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py b/ads/opctl/operator/lowcode/forecast/model/neuralprophet.py
@@ -218,6 +218,7 @@ def _train_model(self, i, s_id, df, model_kwargs):
             logger.debug("===========Done===========")
         except Exception as e:
             self.errors_dict[s_id] = {"model_name": self.spec.model, "error": str(e)}
+            logger.warn(traceback.format_exc())
             raise e
 
     def _build_model(self) -> pd.DataFrame:
diff --git a/ads/opctl/operator/lowcode/forecast/model/prophet.py b/ads/opctl/operator/lowcode/forecast/model/prophet.py
@@ -131,7 +131,8 @@ def _train_model(self, i, series_id, df, model_kwargs):
                 "model_name": self.spec.model,
                 "error": str(e),
             }
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
 
     def _build_model(self) -> pd.DataFrame:
         full_data_dict = self.datasets.get_data_by_series()

Original file line number	Diff line number	Diff line change
`@@ -209,7 +209,8 @@ def _build_model(self) -> pd.DataFrame:`
`209`	`209`	`"model_name": self.spec.model,`
`210`	`210`	`"error": str(e),`
`211`	`211`	`}`
`212`		`- logger.debug(f"Encountered Error: {e}. Skipping.")`
	`212`	`+ logger.warn(f"Encountered Error: {e}. Skipping.")`
	`213`	`+ logger.warn(traceback.format_exc())`
`213`	`214`
`214`	`215`	`logger.debug("===========Done===========")`
`215`	`216`
Original file line number	Diff line number	Diff line change
`@@ -165,7 +165,8 @@ def _train_model(self, data_train, data_test, model_kwargs):`
`165`	`165`	`"model_name": self.spec.model,`
`166`	`166`	`"error": str(e),`
`167`	`167`	`}`
`168`		`- logger.debug(f"Encountered Error: {e}. Skipping.")`
	`168`	`+ logger.warn(f"Encountered Error: {e}. Skipping.")`
	`169`	`+ logger.warn(traceback.format_exc())`
`169`	`170`	`raise e`
`170`	`171`
`171`	`172`	`def _build_model(self) -> pd.DataFrame:`
Original file line number	Diff line number	Diff line change
`@@ -131,7 +131,8 @@ def _train_model(self, i, series_id, df, model_kwargs):`
`131`	`131`	`"model_name": self.spec.model,`
`132`	`132`	`"error": str(e),`
`133`	`133`	`}`
`134`		`- logger.debug(f"Encountered Error: {e}. Skipping.")`
	`134`	`+ logger.warn(f"Encountered Error: {e}. Skipping.")`
	`135`	`+ logger.warn(traceback.format_exc())`
`135`	`136`
`136`	`137`	`def _build_model(self) -> pd.DataFrame:`
`137`	`138`	`full_data_dict = self.datasets.get_data_by_series()`