Odsc 50180/negative r2 (#448)

Author: ahosler

.github/workflows/run-operators-unit-tests.yml

@@ -0,0 +1,59 @@
+name: "[Py3.8][Py3.10] Operators Tests"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - "ads/opctl/operator/**"
+      - "**requirements.txt"
+      - ".github/workflows/run-operators*.yml"
+      - "test-requirements-operators.txt"
+
+# Cancel in progress workflows on pull_requests.
+# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359
+env:
+  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5
+
+jobs:
+  test:
+    name: python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.10.8"]
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+          cache-dependency-path: |
+            pyproject.toml
+            "**requirements.txt"
+            "test-requirements-operators.txt"
+
+      - uses: ./.github/workflows/set-dummy-conf
+        name: "Test config setup"
+
+      - name: "Run Operators Tests"
+        timeout-minutes: 60
+        shell: bash
+        run: |
+          set -x # print commands that are executed
+          $CONDA/bin/conda init
+          source /home/runner/.bashrc
+          pip install -r test-requirements-operators.txt
+          python -m pytest -v -p no:warnings --durations=5 tests/operators

ads/common/oci_client.py

@@ -17,6 +17,7 @@
 from oci.resource_search import ResourceSearchClient
 from oci.secrets import SecretsClient
 from oci.vault import VaultsClient
+
 logger = logging.getLogger(__name__)
 
 
@@ -64,13 +65,14 @@ def _client_impl(self, client):
             "data_labeling_dp": DataLabelingClient,
             "data_labeling_cp": DataLabelingManagementClient,
             "resource_search": ResourceSearchClient,
-            "data_catalog": DataCatalogClient
+            "data_catalog": DataCatalogClient,
         }
         try:
             from oci.feature_store import FeatureStoreClient
+
             client_map["feature_store"] = FeatureStoreClient
         except ImportError:
-            logger.warning("OCI SDK with feature store support is not installed")
+            logger.debug("OCI SDK with feature store support is not installed")
             pass
 
         assert (

ads/opctl/distributed/cmds.py

@@ -4,18 +4,22 @@
 # Copyright (c) 2022, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import os
-import fsspec
-import subprocess
-import yaml
 import json
+import os
 import re
-import docker
+import subprocess
 from configparser import ConfigParser
-
-from ads.common.auth import create_signer, AuthType, AuthContext
 from urllib.parse import urlparse
-from ads.jobs import Job, DataScienceJobRun
+
+import fsspec
+import yaml
+
+from ads.common.auth import AuthContext, AuthType, create_signer
+from ads.common.decorator.runtime_dependency import (
+    OptionalDependency,
+    runtime_dependency,
+)
+from ads.jobs import Job
 from ads.jobs.builders.runtimes.artifact import Artifact
 from ads.opctl.utils import publish_image as publish_image_cmd
 from ads.opctl.utils import run_command
@@ -137,6 +141,7 @@ def show_config_info(job_id, work_dir, cluster_file_name, worker_info, **kwargs)
         )
 
 
+@runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)
 def verify_image(img_name):
     """
     Verify if the input image exists in OCI registry
@@ -151,6 +156,8 @@ def verify_image(img_name):
     -------
 
     """
+    import docker
+
     client = docker.from_env()
     try:
         client.images.get_registry_data(img_name)

ads/opctl/operator/lowcode/forecast/model/arima.py

@@ -87,6 +87,7 @@ def _build_model(self) -> pd.DataFrame:
 
             fitted_values[target] = model.predict_in_sample(X=X_in)
             actual_values[target] = y
+            actual_values[target].index = pd.to_datetime(y.index)
 
             # Build future dataframe
             start_date = y.index.values[-1]
@@ -107,12 +108,7 @@ def _build_model(self) -> pd.DataFrame:
             )
             yhat_clean = pd.DataFrame(yhat, index=yhat.index, columns=["yhat"])
 
-            dt_columns[target] = pd.concat(
-                [
-                    df_encoded[self.spec.datetime_column.name],
-                    pd.Series(yhat_clean.index),
-                ]
-            )
+            dt_columns[target] = df_encoded[self.spec.datetime_column.name]
             conf_int_clean = pd.DataFrame(
                 conf_int, index=yhat.index, columns=["yhat_lower", "yhat_upper"]
             )
@@ -139,10 +135,10 @@ def _build_model(self) -> pd.DataFrame:
         for cat in self.categories:
             output_i = pd.DataFrame()
             output_i["Date"] = dt_columns[f"{col}_{cat}"]
-            output_i = output_i.set_index("Date")
             output_i["Series"] = cat
-            output_i["input_value"] = actual_values[f"{col}_{cat}"]
+            output_i = output_i.set_index("Date")
 
+            output_i["input_value"] = actual_values[f"{col}_{cat}"]
             output_i["fitted_value"] = fitted_values[f"{col}_{cat}"]
             output_i["forecast_value"] = outputs[f"{col}_{cat}"]["yhat"]
             output_i[yhat_upper_name] = outputs[f"{col}_{cat}"]["yhat_upper"]

ads/opctl/operator/lowcode/forecast/model/forecast_datasets.py

@@ -7,7 +7,6 @@
 import pandas as pd
 from ..operator_config import ForecastOperatorConfig
 from .. import utils
-from ads.opctl.operator.lowcode.forecast.utils import default_signer
 from .transformations import Transformations
 from ads.opctl import logger
 import pandas as pd
@@ -40,7 +39,6 @@ def _load_data(self, spec):
         raw_data = utils._load_data(
             filename=spec.historical_data.url,
             format=spec.historical_data.format,
-            storage_options=default_signer(),
             columns=spec.historical_data.columns,
         )
         self.original_user_data = raw_data.copy()
@@ -71,7 +69,6 @@ def _load_data(self, spec):
             additional_data = utils._load_data(
                 filename=spec.additional_data.url,
                 format=spec.additional_data.format,
-                storage_options=default_signer(),
                 columns=spec.additional_data.columns,
             )
             additional_data = data_transformer._sort_by_datetime_col(additional_data)

ads/opctl/operator/lowcode/forecast/utils.py

@@ -17,9 +17,13 @@
     explained_variance_score,
     mean_absolute_percentage_error,
     mean_squared_error,
-    r2_score,
 )
 
+try:
+    from scipy.stats import linregress
+except:
+    from sklearn.metrics import r2_score
+
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.dataset.label_encoder import DataFrameLabelEncoder
 from ads.opctl import logger
@@ -358,7 +362,10 @@ def _build_metrics_df(y_true, y_pred, column_name):
     metrics["sMAPE"] = smape(actual=y_true, predicted=y_pred)
     metrics["MAPE"] = mean_absolute_percentage_error(y_true=y_true, y_pred=y_pred)
     metrics["RMSE"] = np.sqrt(mean_squared_error(y_true=y_true, y_pred=y_pred))
-    metrics["r2"] = r2_score(y_true=y_true, y_pred=y_pred)
+    try:
+        metrics["r2"] = linregress(y_true, y_pred).rvalue ** 2
+    except:
+        metrics["r2"] = r2_score(y_true=y_true, y_pred=y_pred)
     metrics["Explained Variance"] = explained_variance_score(
         y_true=y_true, y_pred=y_pred
     )

docs/source/user_guide/apachespark/setup-installation.rst

@@ -24,7 +24,7 @@ Activate the conda environment to upgrade to the latest ``oracle-ads``
 .. code-block:: shell
 
     conda activate /home/datascience/conda/pyspark30_p37_cpu_v5
-    pip install oracle-ads[data_science, data, opctl] --upgrade
+    pip install "oracle-ads[data_science, data, opctl]" --upgrade
 
 
 Configuring core-site.xml
@@ -200,4 +200,3 @@ Data Flow to access logs bucket, use a policy like:
    ALLOW SERVICE dataflow TO READ objects IN tenancy WHERE target.bucket.name='dataflow-logs'
 
 For more information, see the `Data Flow documentation <https://docs.oracle.com/en-us/iaas/data-flow/using/dfs_getting_started.htm#set_up_admin>`__.
-

docs/source/user_guide/big_data_service/boilerplate/create_conda.rst

@@ -4,6 +4,5 @@ The following are the recommended steps to create a conda environment to connect
 - ``odsc conda install -s pyspark30_p37_cpu_v3``: Install the PySpark conda environment.
 - ``conda activate /home/datascience/conda/pyspark30_p37_cpu_v3``: Activate the PySpark conda environment so that you can modify it.
 - ``pip uninstall oracle_ads``: Uninstall the old ADS package in this environment.
-- ``pip install oracle_ads[bds]``: Install the latest version of ADS that contains BDS support.
+- ``pip install "oracle_ads[bds]"``: Install the latest version of ADS that contains BDS support.
 - ``conda install sasl``: Install ``sasl``.
-

docs/source/user_guide/cli/quickstart.rst

@@ -61,79 +61,79 @@ To work with gradient boosting models, install the ``boosted`` module. This modu
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[boosted]
+    $ python3 -m pip install "oracle-ads[boosted]"
 
 For big data use cases using Oracle Big Data Service (BDS), install the ``bds`` module. It includes the following libraries: ``ibis-framework[impala]``, ``hdfs[kerberos]`` and ``sqlalchemy``.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[bds]
+    $ python3 -m pip install "oracle-ads[bds]"
 
 To work with a broad set of data formats (for example, Excel, Avro, etc.) install the ``data`` module. It includes the following libraries: ``fastavro``, ``openpyxl``, ``pandavro``, ``asteval``, ``datefinder``, ``htmllistparse``, and ``sqlalchemy``.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[data]
+    $ python3 -m pip install "oracle-ads[data]"
 
 To work with geospatial data install the ``geo`` module. It includes the ``geopandas`` and libraries from the ``viz`` module.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[geo]
+    $ python3 -m pip install "oracle-ads[geo]"
 
 Install the ``notebook`` module to use ADS within the Oracle Cloud Infrastructure Data Science service `Notebook Session <https://docs.oracle.com/en-us/iaas/data-science/using/manage-notebook-sessions.htm>`_. This module installs ``ipywidgets`` and ``ipython`` libraries.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[notebook]
+    $ python3 -m pip install "oracle-ads[notebook]"
 
 To work with ONNX-compatible run times and libraries designed to maximize performance and model portability, install the ``onnx`` module. It includes the following libraries, ``onnx``, ``onnxruntime``, ``onnxmltools``, ``skl2onnx``, ``xgboost``, ``lightgbm`` and libraries from the ``viz`` module.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[onnx]
+    $ python3 -m pip install "oracle-ads[onnx]"
 
 For infrastructure tasks, install the ``opctl`` module. It includes the following libraries, ``oci-cli``, ``docker``, ``conda-pack``, ``nbconvert``, ``nbformat``, and ``inflection``.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[opctl]
+    $ python3 -m pip install "oracle-ads[opctl]"
 
 For hyperparameter optimization tasks install the ``optuna`` module. It includes the ``optuna`` and libraries from the ``viz`` module.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[optuna]
+    $ python3 -m pip install "oracle-ads[optuna]"
 
 For Spark tasks install the ``spark`` module.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[spark]
+    $ python3 -m pip install "oracle-ads[spark]"
 
 Install the ``tensorflow`` module to include ``tensorflow`` and libraries from the ``viz`` module.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[tensorflow]
+    $ python3 -m pip install "oracle-ads[tensorflow]"
 
 For text related tasks, install the ``text`` module. This will include the ``wordcloud``, ``spacy`` libraries.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[text]
+    $ python3 -m pip install "oracle-ads[text]"
 
 Install the ``torch`` module to include ``pytorch`` and libraries from the ``viz`` module.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[torch]
+    $ python3 -m pip install "oracle-ads[torch]"
 
 Install the ``viz`` module to include libraries for visualization tasks. Some of the key packages are ``bokeh``, ``folium``, ``seaborn`` and related packages.
 
 .. code-block:: bash
 
-    $ python3 -m pip install oracle-ads[viz]
+    $ python3 -m pip install "oracle-ads[viz]"
 
 See ``pyproject.toml`` file ``[project.optional-dependencies]`` section for full list of modules and its list of extra libraries.
 
@@ -143,4 +143,4 @@ Multiple extra dependencies can be installed together. For example:
 
 .. code-block:: bash
 
-    $ python3 -m pip install  oracle-ads[notebook,viz,text]
+    $ python3 -m pip install  "oracle-ads[notebook,viz,text]"

docs/source/user_guide/operators/forecasting_operator/faq.rst

@@ -9,3 +9,7 @@ More details in the documentation here: https://docs.oracle.com/en-us/iaas/tools
 **How do I learn More about AutoTS?**
 
 More details in the documentation here: https://winedarksea.github.io/AutoTS/build/html/source/tutorial.html
+
+**Pip Install Failing with "ERROR: No matching distribution found for oracle-automlx==23.2.3; extra == "forecast""**
+
+Automlx only supports Python<=3.8, != 3.9, <= 3.10.7 . If you are builing in Python 3.9 or 3.10.8+, no automlx distribution will be available. It's recommended to use the conda pack available through Notebook Sessions, or to use Python 3.8