Automlx | Logging errors in error.csv when all forecast values are NaN. (#932)

Author: ahosler

ads/opctl/operator/lowcode/forecast/model/arima.py

@@ -126,8 +126,12 @@ def _train_model(self, i, s_id, df, model_kwargs):
 
             logger.debug("===========Done===========")
         except Exception as e:
-            self.errors_dict[s_id] = {"model_name": self.spec.model, "error": str(e)}
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            self.errors_dict[s_id] = {
+                "model_name": self.spec.model,
+                "error": str(e),
+                "error_trace": traceback.format_exc()}
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
 
     def _build_model(self) -> pd.DataFrame:
         full_data_dict = self.datasets.get_data_by_series()

ads/opctl/operator/lowcode/forecast/model/automlx.py

@@ -149,6 +149,9 @@ def _build_model(self) -> pd.DataFrame:
                 if f"{target}_ci_lower" not in summary_frame:
                     summary_frame[f"{target}_ci_lower"] = np.NAN
 
+                if summary_frame[target].isna().all():
+                    raise ValueError("The forecasts are completely NaN")
+
                 self.forecast_output.populate_series_output(
                     series_id=s_id,
                     fit_val=fitted_values,
@@ -167,8 +170,10 @@ def _build_model(self) -> pd.DataFrame:
                 self.errors_dict[s_id] = {
                     "model_name": self.spec.model,
                     "error": str(e),
+                    "error_trace": traceback.format_exc()
                 }
-                logger.debug(f"Encountered Error: {e}. Skipping.")
+                logger.warn(f"Encountered Error: {e}. Skipping.")
+                logger.warn(traceback.format_exc())
 
         logger.debug("===========Forecast Generated===========")
 

ads/opctl/operator/lowcode/forecast/model/autots.py

@@ -208,8 +208,10 @@ def _build_model(self) -> pd.DataFrame:
                 self.errors_dict[s_id] = {
                     "model_name": self.spec.model,
                     "error": str(e),
+                    "error_trace": traceback.format_exc()
                 }
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
 
         logger.debug("===========Done===========")
 

ads/opctl/operator/lowcode/forecast/model/ml_forecast.py

@@ -164,8 +164,10 @@ def _train_model(self, data_train, data_test, model_kwargs):
             self.errors_dict[self.spec.model] = {
                 "model_name": self.spec.model,
                 "error": str(e),
+                "error_trace": traceback.format_exc()
             }
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
             raise e
 
     def _build_model(self) -> pd.DataFrame:

ads/opctl/operator/lowcode/forecast/model/neuralprophet.py

@@ -217,7 +217,12 @@ def _train_model(self, i, s_id, df, model_kwargs):
 
             logger.debug("===========Done===========")
         except Exception as e:
-            self.errors_dict[s_id] = {"model_name": self.spec.model, "error": str(e)}
+            self.errors_dict[s_id] = {
+                "model_name": self.spec.model,
+                "error": str(e),
+                "error_trace": traceback.format_exc()
+            }
+            logger.warn(traceback.format_exc())
             raise e
 
     def _build_model(self) -> pd.DataFrame:

ads/opctl/operator/lowcode/forecast/model/prophet.py

@@ -130,8 +130,10 @@ def _train_model(self, i, series_id, df, model_kwargs):
             self.errors_dict[series_id] = {
                 "model_name": self.spec.model,
                 "error": str(e),
+                "error_trace": traceback.format_exc()
             }
-            logger.debug(f"Encountered Error: {e}. Skipping.")
+            logger.warn(f"Encountered Error: {e}. Skipping.")
+            logger.warn(traceback.format_exc())
 
     def _build_model(self) -> pd.DataFrame:
         full_data_dict = self.datasets.get_data_by_series()