Merge branch 'main' into feature/recommender_v1

Author: ahosler

.github/workflows/run-operators-unit-tests.yml

@@ -27,7 +27,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-default_setup.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][Py3.9][Py3.10] tests/unitary/default_setup/**"
+name: "[Py3.8-3.11] - Default Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/run-unittests-py38-cov-report.yml

@@ -1,4 +1,4 @@
-name: "[Py3.8][COV REPORT] tests/unitary/**"
+name: "[Py3.8][COV REPORT] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -44,6 +44,7 @@ jobs:
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore  tests/unitary/with_extras/operator/feature-store \
               --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/hpo
           - name: "slow_tests"
@@ -79,7 +80,7 @@ jobs:
         run: |
           pip install -e ".[feature-store-marketplace]"
       - name: "Run unitary tests folder with maximum ADS dependencies"
-        timeout-minutes: 40
+        timeout-minutes: 60
         shell: bash
         env:
           CONDA_PREFIX: /usr/share/miniconda
@@ -188,6 +189,7 @@ jobs:
           fi
 
       - name: "Add comment with cov diff to PR"
+        continue-on-error: true
         uses: actions/github-script@v7
         if: github.event_name == 'pull_request'
         with:

.github/workflows/run-unittests-py39-py310.yml

@@ -1,4 +1,4 @@
-name: "[Py3.9][Py3.10] - tests/unitary/**"
+name: "[Py3.9-3.11] - All Unit Tests"
 
 on:
   workflow_dispatch:
@@ -33,7 +33,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
         name: ["unitary", "slow_tests"]
         include:
           - name: "unitary"
@@ -46,6 +46,7 @@ jobs:
             ignore-path: |
               --ignore tests/unitary/with_extras/model \
               --ignore tests/unitary/with_extras/feature_store \
+              --ignore tests/unitary/with_extras/operator/feature-store \
               --ignore tests/unitary/with_extras/operator/forecast \
               --ignore tests/unitary/with_extras/operator/pii \
               --ignore tests/unitary/with_extras/hpo
@@ -85,7 +86,7 @@ jobs:
           tests/unitary/with_extras/hpo
 
       - name: "Run unitary tests folder with maximum ADS dependencies"
-        timeout-minutes: 30
+        timeout-minutes: 60
         shell: bash
         env:
           CONDA_PREFIX: /usr/share/miniconda

.github/workflows/test-env-setup/action.yml

@@ -7,6 +7,11 @@ runs:
   using: composite
   steps:
     - shell: bash
+      env:
+        # torch > v2.1.0 brings nvidia dependency by default, we want to install torch for cpu for tests.
+        # Fot that --index-url https://download.pytorch.org/whl/cpu needed - https://pytorch.org/get-started/locally/.
+        # Setting env variable here instead of flag --index-url (docs: https://pip.pypa.io/en/stable/cli/pip_install/):
+        PIP_EXTRA_INDEX_URL: "https://download.pytorch.org/whl/cpu"
       run: |
         set -x # print commands that are executed
 

.pre-commit-config.yaml

@@ -1,7 +1,19 @@
 repos:
+    # ruff
+    - repo: https://github.com/astral-sh/ruff-pre-commit
+      rev: v0.5.0
+      hooks:
+          - id: ruff
+            types_or: [ python, pyi, jupyter ]
+            args: [ --fix ]
+            files: ^ads
+            exclude: ^docs/
+          - id: ruff-format
+            types_or: [ python, pyi, jupyter ]
+            exclude: ^docs/
     # Standard hooks
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.4.0
+      rev: v4.6.0
       hooks:
           - id: check-ast
             exclude: ^docs/
@@ -20,12 +32,6 @@ repos:
           - id: trailing-whitespace
             args: [--markdown-linebreak-ext=md]
             exclude: ^docs/
-    # Black, the code formatter, natively supports pre-commit
-    - repo: https://github.com/psf/black
-      rev: 23.3.0
-      hooks:
-          - id: black
-            exclude: ^docs/
     # Regex based rst files common mistakes detector
     - repo: https://github.com/pre-commit/pygrep-hooks
       rev: v1.10.0
@@ -36,7 +42,7 @@ repos:
             files: ^docs/
     # Hardcoded secrets and ocids detector
     - repo: https://github.com/gitleaks/gitleaks
-      rev: v8.17.0
+      rev: v8.18.4
       hooks:
           - id: gitleaks
             exclude: .github/workflows/reusable-actions/set-dummy-conf.yml

ads/aqua/__init__.py

@@ -7,7 +7,7 @@
 import os
 
 from ads import logger, set_auth
-from ads.aqua.utils import fetch_service_compartment
+from ads.aqua.common.utils import fetch_service_compartment
 from ads.config import OCI_RESOURCE_PRINCIPAL_VERSION
 
 ENV_VAR_LOG_LEVEL = "ADS_AQUA_LOG_LEVEL"

ads/aqua/base.py renamed to ads/aqua/app.py

@@ -4,22 +4,23 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
+from dataclasses import fields
 from typing import Dict, Union
 
 import oci
 from oci.data_science.models import UpdateModelDetails, UpdateModelProvenanceDetails
 
 from ads import set_auth
 from ads.aqua import logger
-from ads.aqua.data import Tags
-from ads.aqua.exception import AquaRuntimeError, AquaValueError
-from ads.aqua.utils import (
-    UNKNOWN,
+from ads.aqua.common.enums import Tags
+from ads.aqua.common.errors import AquaRuntimeError, AquaValueError
+from ads.aqua.common.utils import (
     _is_valid_mvs,
     get_artifact_path,
     is_valid_ocid,
     load_config,
 )
+from ads.aqua.constants import UNKNOWN
 from ads.common import oci_client as oc
 from ads.common.auth import default_signer
 from ads.common.utils import extract_region
@@ -160,7 +161,7 @@ def create_model_version_set(
         """
         # TODO: tag should be selected based on which operation (eval/FT) invoke this method
         #   currently only used by fine-tuning flow.
-        tag = Tags.AQUA_FINE_TUNING.value
+        tag = Tags.AQUA_FINE_TUNING
 
         if not model_version_set_id:
             try:
@@ -277,8 +278,8 @@ def get_config(self, model_id: str, config_file_name: str) -> Dict:
         oci_model = self.ds_client.get_model(model_id).data
         oci_aqua = (
             (
-                Tags.AQUA_TAG.value in oci_model.freeform_tags
-                or Tags.AQUA_TAG.value.lower() in oci_model.freeform_tags
+                Tags.AQUA_TAG in oci_model.freeform_tags
+                or Tags.AQUA_TAG.lower() in oci_model.freeform_tags
             )
             if oci_model.freeform_tags
             else False
@@ -302,7 +303,15 @@ def get_config(self, model_id: str, config_file_name: str) -> Dict:
                 config_file_name=config_file_name,
             )
         except:
-            pass
+            # todo: temp fix for issue related to config load for byom models, update logic to choose the right path
+            try:
+                config_path = f"{artifact_path.rstrip('/')}/config/"
+                config = load_config(
+                    config_path,
+                    config_file_name=config_file_name,
+                )
+            except:
+                pass
 
         if not config:
             logger.error(
@@ -319,3 +328,22 @@ def telemetry(self):
                 bucket=AQUA_TELEMETRY_BUCKET, namespace=AQUA_TELEMETRY_BUCKET_NS
             )
         return self._telemetry
+
+
+class CLIBuilderMixin:
+    """
+    CLI builder from API interface. To be used with the DataClass only.
+    """
+
+    def build_cli(self) -> str:
+        """
+        Method to turn the dataclass attributes to CLI
+        """
+        cmd = f"ads aqua {self._command}"
+        params = [
+            f"--{field.name} {getattr(self,field.name)}"
+            for field in fields(self.__class__)
+            if getattr(self, field.name)
+        ]
+        cmd = f"{cmd} {' '.join(params)}"
+        return cmd

ads/aqua/cli.py

@@ -4,20 +4,20 @@
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 import os
-import sys
 
 from ads.aqua import (
     ENV_VAR_LOG_LEVEL,
-    set_log_level,
     ODSC_MODEL_COMPARTMENT_OCID,
     logger,
+    set_log_level,
 )
-from ads.aqua.deployment import AquaDeploymentApp
+from ads.aqua.common.errors import AquaCLIError, AquaConfigError
 from ads.aqua.evaluation import AquaEvaluationApp
-from ads.aqua.finetune import AquaFineTuningApp
+from ads.aqua.finetuning import AquaFineTuningApp
 from ads.aqua.model import AquaModelApp
-from ads.config import NB_SESSION_OCID
+from ads.aqua.modeldeployment import AquaDeploymentApp
 from ads.common.utils import LOG_LEVELS
+from ads.config import NB_SESSION_OCID
 
 
 class AquaCommand:
@@ -35,6 +35,8 @@ class AquaCommand:
 
     def __init__(
         self,
+        debug: bool = None,
+        verbose: bool = None,
         log_level: str = os.environ.get(ENV_VAR_LOG_LEVEL, "ERROR").upper(),
     ):
         """
@@ -44,24 +46,64 @@ def __init__(
         -----
         log_level (str):
             Sets the logging level for the application.
-            Default is retrieved from environment variable `LOG_LEVEL`,
+            Default is retrieved from environment variable `ADS_AQUA_LOG_LEVEL`,
             or 'ERROR' if not set. Example values include 'DEBUG', 'INFO',
             'WARNING', 'ERROR', and 'CRITICAL'.
+        debug (bool):
+            Sets the logging level for the application to `DEBUG`.
+        verbose (bool):
+            Sets the logging level for the application to `INFO`.
+
+        Raises
+        ------
+        AquaCLIError:
+            When `--verbose` and `--debug` being used together.
+            When missing required `ODSC_MODEL_COMPARTMENT_OCID` env var.
         """
-        if log_level.upper() not in LOG_LEVELS:
-            logger.error(
-                f"Log level should be one of {LOG_LEVELS}. Setting default to ERROR."
+        if verbose is not None and debug is not None:
+            raise AquaCLIError(
+                "Cannot use `--debug` and `--verbose` at the same time. "
+                "Please select either `--debug` for `DEBUG` level logging or "
+                "`--verbose` for `INFO` level logging."
             )
-            log_level = "ERROR"
-        set_log_level(log_level)
-        # gracefully exit if env var is not set
+        elif verbose is not None:
+            self._validate_value("--verbose", verbose)
+            aqua_log_level = "INFO"
+        elif debug is not None:
+            self._validate_value("--debug", debug)
+            aqua_log_level = "DEBUG"
+        else:
+            if log_level.upper() not in LOG_LEVELS:
+                logger.warning(
+                    f"Log level should be one of {LOG_LEVELS}. Setting default to ERROR."
+                )
+                log_level = "ERROR"
+            aqua_log_level = log_level.upper()
+
+        set_log_level(aqua_log_level)
+
         if not ODSC_MODEL_COMPARTMENT_OCID:
-            logger.debug(
-                "ODSC_MODEL_COMPARTMENT_OCID environment variable is not set for Aqua."
-            )
             if NB_SESSION_OCID:
-                logger.error(
+                raise AquaConfigError(
                     f"Aqua is not available for the notebook session {NB_SESSION_OCID}. For more information, "
                     f"please refer to the documentation."
                 )
-            sys.exit(1)
+            raise AquaConfigError(
+                "ODSC_MODEL_COMPARTMENT_OCID environment variable is not set for Aqua."
+            )
+
+    @staticmethod
+    def _validate_value(flag, value):
+        """Check if the given value for bool flag is valid.
+
+        Raises
+        ------
+        AquaCLIError:
+            When the given value for bool flag is invalid.
+        """
+        if value not in [True, False]:
+            raise AquaCLIError(
+                f"Invalid input `{value}` for flag: {flag}, a boolean value is required. "
+                "If you intend to chain a function call to the result, please separate the "
+                "flag and the subsequent function call with separator `-`."
+            )

ads/aqua/common/__init__.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/