Merge branch 'main' into ODSC-64829/enhanced-auto-select-report

Author: prasankh

ads/aqua/extension/ui_handler.py

@@ -68,6 +68,8 @@ def get(self, id=""):
             return self.list_buckets()
         elif paths.startswith("aqua/job/shapes"):
             return self.list_job_shapes()
+        elif paths.startswith("aqua/modeldeployment/shapes"):
+            return self.list_model_deployment_shapes()
         elif paths.startswith("aqua/vcn"):
             return self.list_vcn()
         elif paths.startswith("aqua/subnets"):
@@ -160,6 +162,15 @@ def list_job_shapes(self, **kwargs):
             AquaUIApp().list_job_shapes(compartment_id=compartment_id, **kwargs)
         )
 
+    def list_model_deployment_shapes(self, **kwargs):
+        """Lists model deployment shapes available in the specified compartment."""
+        compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
+        return self.finish(
+            AquaUIApp().list_model_deployment_shapes(
+                compartment_id=compartment_id, **kwargs
+            )
+        )
+
     def list_vcn(self, **kwargs):
         """Lists the virtual cloud networks (VCNs) in the specified compartment."""
         compartment_id = self.get_argument("compartment_id", default=COMPARTMENT_OCID)
@@ -255,8 +266,9 @@ def post(self, *args, **kwargs):
 __handlers__ = [
     ("logging/?([^/]*)", AquaUIHandler),
     ("compartments/?([^/]*)", AquaUIHandler),
-    # TODO: change url to evaluation/experiements/?([^/]*)
+    # TODO: change url to evaluation/experiments/?([^/]*)
     ("experiment/?([^/]*)", AquaUIHandler),
+    ("modeldeployment/?([^/]*)", AquaUIHandler),
     ("versionsets/?([^/]*)", AquaUIHandler),
     ("buckets/?([^/]*)", AquaUIHandler),
     ("job/shapes/?([^/]*)", AquaUIHandler),

ads/aqua/modeldeployment/deployment.py

@@ -185,7 +185,7 @@ def create(
                 tags[tag] = aqua_model.freeform_tags[tag]
 
         tags.update({Tags.AQUA_MODEL_NAME_TAG: aqua_model.display_name})
-        tags.update({Tags.TASK: aqua_model.freeform_tags.get(Tags.TASK, None)})
+        tags.update({Tags.TASK: aqua_model.freeform_tags.get(Tags.TASK, UNKNOWN)})
 
         # Set up info to get deployment config
         config_source_id = model_id
@@ -533,16 +533,22 @@ def list(self, **kwargs) -> List["AquaDeployment"]:
         return results
 
     @telemetry(entry_point="plugin=deployment&action=delete", name="aqua")
-    def delete(self,model_deployment_id:str):
-        return self.ds_client.delete_model_deployment(model_deployment_id=model_deployment_id).data
+    def delete(self, model_deployment_id: str):
+        return self.ds_client.delete_model_deployment(
+            model_deployment_id=model_deployment_id
+        ).data
 
-    @telemetry(entry_point="plugin=deployment&action=deactivate",name="aqua")
-    def deactivate(self,model_deployment_id:str):
-        return self.ds_client.deactivate_model_deployment(model_deployment_id=model_deployment_id).data
+    @telemetry(entry_point="plugin=deployment&action=deactivate", name="aqua")
+    def deactivate(self, model_deployment_id: str):
+        return self.ds_client.deactivate_model_deployment(
+            model_deployment_id=model_deployment_id
+        ).data
 
-    @telemetry(entry_point="plugin=deployment&action=activate",name="aqua")
-    def activate(self,model_deployment_id:str):
-        return self.ds_client.activate_model_deployment(model_deployment_id=model_deployment_id).data
+    @telemetry(entry_point="plugin=deployment&action=activate", name="aqua")
+    def activate(self, model_deployment_id: str):
+        return self.ds_client.activate_model_deployment(
+            model_deployment_id=model_deployment_id
+        ).data
 
     @telemetry(entry_point="plugin=deployment&action=get", name="aqua")
     def get(self, model_deployment_id: str, **kwargs) -> "AquaDeploymentDetail":

ads/aqua/ui.py

@@ -481,12 +481,12 @@ def _is_bucket_versioned(response):
 
     @telemetry(entry_point="plugin=ui&action=list_job_shapes", name="aqua")
     def list_job_shapes(self, **kwargs) -> list:
-        """Lists all availiable job shapes for the specified compartment.
+        """Lists all available job shapes for the specified compartment.
 
         Parameters
         ----------
         **kwargs
-            Addtional arguments, such as `compartment_id`,
+            Additional arguments, such as `compartment_id`,
             for `list_job_shapes <https://docs.oracle.com/en-us/iaas/tools/python/2.122.0/api/data_science/client/oci.data_science.DataScienceClient.html#oci.data_science.DataScienceClient.list_job_shapes>`_
 
         Returns
@@ -500,6 +500,28 @@ def list_job_shapes(self, **kwargs) -> list:
         ).data
         return sanitize_response(oci_client=self.ds_client, response=res)
 
+    @telemetry(entry_point="plugin=ui&action=list_model_deployment_shapes", name="aqua")
+    def list_model_deployment_shapes(self, **kwargs) -> list:
+        """Lists all available shapes for model deployment in the specified compartment.
+
+        Parameters
+        ----------
+        **kwargs
+            Additional arguments, such as `compartment_id`,
+            for `list_model_deployment_shapes <https://docs.oracle.com/en-us/iaas/api/#/en/data-science/20190101/ModelDeploymentShapeSummary/ListModelDeploymentShapes>`_
+
+        Returns
+        -------
+            str has json representation of `oci.data_science.models.ModelDeploymentShapeSummary`."""
+        compartment_id = kwargs.pop("compartment_id", COMPARTMENT_OCID)
+        logger.info(
+            f"Loading model deployment shape summary from compartment: {compartment_id}"
+        )
+        res = self.ds_client.list_model_deployment_shapes(
+            compartment_id=compartment_id, **kwargs
+        ).data
+        return sanitize_response(oci_client=self.ds_client, response=res)
+
     @telemetry(entry_point="plugin=ui&action=list_vcn", name="aqua")
     def list_vcn(self, **kwargs) -> list:
         """Lists the virtual cloud networks (VCNs) in the specified compartment.

ads/opctl/operator/common/utils.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
@@ -18,23 +17,26 @@
 from cerberus import Validator
 
 from ads.opctl import logger, utils
-from ads.opctl.operator import __operators__
 
 CONTAINER_NETWORK = "CONTAINER_NETWORK"
 
 
 class OperatorValidator(Validator):
     """The custom validator class."""
 
-    pass
+    def validate(self, obj_dict, **kwargs):
+        # Model should be case insensitive
+        if "model" in obj_dict["spec"]:
+            obj_dict["spec"]["model"] = str(obj_dict["spec"]["model"]).lower()
+        return super().validate(obj_dict, **kwargs)
 
 
 def create_output_folder(name):
     output_folder = name
     protocol = fsspec.utils.get_protocol(output_folder)
     storage_options = {}
     if protocol != "file":
-        storage_options = auth or default_signer()
+        storage_options = default_signer()
 
     fs = fsspec.filesystem(protocol, **storage_options)
     name_suffix = 1

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -166,9 +166,8 @@ def generate_report(self):
         yaml_appendix = rc.Yaml(self.config.to_dict())
         summary = rc.Block(
             rc.Group(
-                rc.Text(
-                    f"You selected the **`{self.spec.model}`** model.\n{model_description.text}\n"
-                ),
+                rc.Text(f"You selected the **`{self.spec.model}`** model.\n"),
+                model_description,
                 rc.Text(
                     "Based on your dataset, you could have also selected "
                     f"any of the models: `{'`, `'.join(SupportedModels.keys() if self.spec.datetime_column else NonTimeADSupportedModels.keys())}`."

ads/opctl/operator/lowcode/anomaly/model/factory.py

@@ -26,9 +26,9 @@ class UnSupportedModelError(Exception):
 
     def __init__(self, operator_config: AnomalyOperatorConfig, model_type: str):
         supported_models = (
-            SupportedModels.values
+            SupportedModels.values()
             if operator_config.spec.datetime_column
-            else NonTimeADSupportedModels.values
+            else NonTimeADSupportedModels.values()
         )
         message = (
             f"Model: `{model_type}` is not supported. "

ads/opctl/operator/lowcode/common/transformations.py

@@ -1,18 +1,19 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
-# Copyright (c) 2023 Oracle and/or its affiliates.
+# Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
+from abc import ABC
+
+import pandas as pd
+
 from ads.opctl import logger
+from ads.opctl.operator.lowcode.common.const import DataColumns
 from ads.opctl.operator.lowcode.common.errors import (
-    InvalidParameterError,
     DataMismatchError,
+    InvalidParameterError,
 )
-from ads.opctl.operator.lowcode.common.const import DataColumns
 from ads.opctl.operator.lowcode.common.utils import merge_category_columns
-import pandas as pd
-from abc import ABC
 
 
 class Transformations(ABC):
@@ -58,6 +59,7 @@ def run(self, data):
 
         """
         clean_df = self._remove_trailing_whitespace(data)
+        # clean_df = self._normalize_column_names(clean_df)
         if self.name == "historical_data":
             self._check_historical_dataset(clean_df)
         clean_df = self._set_series_id_column(clean_df)
@@ -95,8 +97,11 @@ def run(self, data):
     def _remove_trailing_whitespace(self, df):
         return df.apply(lambda x: x.str.strip() if x.dtype == "object" else x)
 
+    # def _normalize_column_names(self, df):
+    #     return df.rename(columns=lambda x: re.sub("[^A-Za-z0-9_]+", "", x))
+
     def _set_series_id_column(self, df):
-        self._target_category_columns_map = dict()
+        self._target_category_columns_map = {}
         if not self.target_category_columns:
             df[DataColumns.Series] = "Series 1"
             self.has_artificial_series = True
@@ -125,10 +130,10 @@ def _format_datetime_col(self, df):
             df[self.dt_column_name] = pd.to_datetime(
                 df[self.dt_column_name], format=self.dt_column_format
             )
-        except:
+        except Exception as ee:
             raise InvalidParameterError(
                 f"Unable to determine the datetime type for column: {self.dt_column_name} in dataset: {self.name}. Please specify the format explicitly. (For example adding 'format: %d/%m/%Y' underneath 'name: {self.dt_column_name}' in the datetime_column section of the yaml file if you haven't already. For reference, here is the first datetime given: {df[self.dt_column_name].values[0]}"
-            )
+            ) from ee
         return df
 
     def _set_multi_index(self, df):
@@ -242,7 +247,6 @@ def _check_historical_dataset(self, df):
                 "Class": "A",
                 "Num": 2
             },
-            
         }
     """
 

ads/opctl/operator/lowcode/common/utils.py

@@ -1,42 +1,32 @@
 #!/usr/bin/env python
-# -*- coding: utf-8 -*--
 
 # Copyright (c) 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
-import argparse
 import logging
 import os
 import shutil
 import sys
 import tempfile
-import time
-from string import Template
-from typing import Any, Dict, List, Tuple
-import pandas as pd
-from ads.opctl import logger
-import oracledb
+from typing import List, Union
 
 import fsspec
-import yaml
-from typing import Union
+import oracledb
+import pandas as pd
 
+from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
+from ads.opctl.operator.common.operator_config import OutputDirectory
 from ads.opctl.operator.lowcode.common.errors import (
-    InputDataError,
     InvalidParameterError,
-    PermissionsError,
-    DataMismatchError,
 )
-from ads.opctl.operator.common.operator_config import OutputDirectory
-from ads.common.object_storage_details import ObjectStorageDetails
 from ads.secrets import ADBSecretKeeper
 
 
 def call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
-    if fsspec.utils.get_protocol(filename) == "file":
-        return pd_fn(filename, **kwargs)
-    elif fsspec.utils.get_protocol(filename) in ["http", "https"]:
+    if fsspec.utils.get_protocol(filename) == "file" or fsspec.utils.get_protocol(
+        filename
+    ) in ["http", "https"]:
         return pd_fn(filename, **kwargs)
 
     storage_options = storage_options or (
@@ -48,7 +38,7 @@ def call_pandas_fsspec(pd_fn, filename, storage_options, **kwargs):
 
 def load_data(data_spec, storage_options=None, **kwargs):
     if data_spec is None:
-        raise InvalidParameterError(f"No details provided for this data source.")
+        raise InvalidParameterError("No details provided for this data source.")
     filename = data_spec.url
     format = data_spec.format
     columns = data_spec.columns
@@ -67,7 +57,7 @@ def load_data(data_spec, storage_options=None, **kwargs):
         if not format:
             _, format = os.path.splitext(filename)
             format = format[1:]
-        if format in ["json", "clipboard", "excel", "csv", "feather", "hdf"]:
+        if format in ["json", "clipboard", "excel", "csv", "feather", "hdf", "parquet"]:
             read_fn = getattr(pd, f"read_{format}")
             data = call_pandas_fsspec(
                 read_fn, filename, storage_options=storage_options
@@ -84,19 +74,31 @@ def load_data(data_spec, storage_options=None, **kwargs):
         with tempfile.TemporaryDirectory() as temp_dir:
             if vault_secret_id is not None:
                 try:
-                    with ADBSecretKeeper.load_secret(vault_secret_id, wallet_dir=temp_dir) as adwsecret:
-                        if 'wallet_location' in adwsecret and 'wallet_location' not in connect_args:
-                            shutil.unpack_archive(adwsecret["wallet_location"], temp_dir)
-                            connect_args['wallet_location'] = temp_dir
-                        if 'user_name' in adwsecret and 'user' not in connect_args:
-                            connect_args['user'] = adwsecret['user_name']
-                        if 'password' in adwsecret and 'password' not in connect_args:
-                            connect_args['password'] = adwsecret['password']
-                        if 'service_name' in adwsecret and 'service_name' not in connect_args:
-                            connect_args['service_name'] = adwsecret['service_name']
+                    with ADBSecretKeeper.load_secret(
+                        vault_secret_id, wallet_dir=temp_dir
+                    ) as adwsecret:
+                        if (
+                            "wallet_location" in adwsecret
+                            and "wallet_location" not in connect_args
+                        ):
+                            shutil.unpack_archive(
+                                adwsecret["wallet_location"], temp_dir
+                            )
+                            connect_args["wallet_location"] = temp_dir
+                        if "user_name" in adwsecret and "user" not in connect_args:
+                            connect_args["user"] = adwsecret["user_name"]
+                        if "password" in adwsecret and "password" not in connect_args:
+                            connect_args["password"] = adwsecret["password"]
+                        if (
+                            "service_name" in adwsecret
+                            and "service_name" not in connect_args
+                        ):
+                            connect_args["service_name"] = adwsecret["service_name"]
 
                 except Exception as e:
-                    raise Exception(f"Could not retrieve database credentials from vault {vault_secret_id}: {e}")
+                    raise Exception(
+                        f"Could not retrieve database credentials from vault {vault_secret_id}: {e}"
+                    )
 
             con = oracledb.connect(**connect_args)
             if table_name is not None:
@@ -105,11 +107,11 @@ def load_data(data_spec, storage_options=None, **kwargs):
                 data = pd.read_sql(sql, con)
             else:
                 raise InvalidParameterError(
-                    f"Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
+                    "Database `connect_args` provided without sql query or table name. Please specify either `sql` or `table_name`."
                 )
     else:
         raise InvalidParameterError(
-            f"No filename/url provided, and no connect_args provided. Please specify one of these if you want to read data from a file or a database respectively."
+            "No filename/url provided, and no connect_args provided. Please specify one of these if you want to read data from a file or a database respectively."
         )
     if columns:
         # keep only these columns, done after load because only CSV supports stream filtering
@@ -232,7 +234,7 @@ def human_time_friendly(seconds):
             accumulator.append(
                 "{} {}{}".format(int(amount), unit, "" if amount == 1 else "s")
             )
-    accumulator.append("{} secs".format(round(seconds, 2)))
+    accumulator.append(f"{round(seconds, 2)} secs")
     return ", ".join(accumulator)
 
 
@@ -248,9 +250,7 @@ def find_output_dirname(output_dir: OutputDirectory):
         unique_output_dir = f"{output_dir}_{counter}"
         counter += 1
     logger.warn(
-        "Since the output directory was not specified, the output will be saved to {} directory.".format(
-            unique_output_dir
-        )
+        f"Since the output directory was not specified, the output will be saved to {unique_output_dir} directory."
     )
     return unique_output_dir