Merge branch 'main' into spavlusieva-patch-1

Author: qiuosier

ads/cli.py

@@ -9,7 +9,6 @@
 
 import fire
 from ads.common import logger
-from ads.aqua.cli import AquaCommand
 
 try:
     import click
@@ -73,6 +72,8 @@ def _SeparateFlagArgs(args):
 
 def cli():
     if len(sys.argv) > 1 and sys.argv[1] == "aqua":
+        from ads.aqua.cli import AquaCommand
+
         fire.Fire(AquaCommand, command=sys.argv[2:], name="ads aqua")
     else:
         click_cli()

ads/opctl/operator/lowcode/anomaly/model/anomaly_dataset.py

@@ -55,6 +55,10 @@ def __init__(self, spec: AnomalyOperatorSpec):
             self.X_valid_dict = self.valid_data.X_valid_dict
             self.y_valid_dict = self.valid_data.y_valid_dict
 
+    # Returns raw data based on the series_id i.e; the merged target_category_column value
+    def get_raw_data_by_cat(self, category):
+        return self._data.get_raw_data_by_cat(category)
+
 
 class AnomalyOutput:
     def __init__(self, date_column):
@@ -93,38 +97,28 @@ def get_outliers_by_cat(self, category: str, data: pd.DataFrame):
             outliers = pd.merge(outliers, scores, on=self.date_column, how="inner")
         return outliers
 
-    def get_inliers(self, data):
+    def get_inliers(self, datasets):
         inliers = pd.DataFrame()
 
         for category in self.list_categories():
             inliers = pd.concat(
                 [
                     inliers,
-                    self.get_inliers_by_cat(
-                        category,
-                        data[data[OutputColumns.Series] == category]
-                        .reset_index(drop=True)
-                        .drop(OutputColumns.Series, axis=1),
-                    ),
+                    self.get_inliers_by_cat(category, datasets.get_raw_data_by_cat(category)),
                 ],
                 axis=0,
                 ignore_index=True,
             )
         return inliers
 
-    def get_outliers(self, data):
+    def get_outliers(self, datasets):
         outliers = pd.DataFrame()
 
         for category in self.list_categories():
             outliers = pd.concat(
                 [
                     outliers,
-                    self.get_outliers_by_cat(
-                        category,
-                        data[data[OutputColumns.Series] == category]
-                        .reset_index(drop=True)
-                        .drop(OutputColumns.Series, axis=1),
-                    ),
+                    self.get_outliers_by_cat(category, datasets.get_raw_data_by_cat(category)),
                 ],
                 axis=0,
                 ignore_index=True,

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -272,15 +272,15 @@ def _save_report(
                     f2.write(f1.read())
 
         if self.spec.generate_inliers:
-            inliers = anomaly_output.get_inliers(self.datasets.data)
+            inliers = anomaly_output.get_inliers(self.datasets)
             write_data(
                 data=inliers,
                 filename=os.path.join(unique_output_dir, self.spec.inliers_filename),
                 format="csv",
                 storage_options=storage_options,
             )
 
-        outliers = anomaly_output.get_outliers(self.datasets.data)
+        outliers = anomaly_output.get_outliers(self.datasets)
         write_data(
             data=outliers,
             filename=os.path.join(unique_output_dir, self.spec.outliers_filename),

ads/opctl/operator/lowcode/common/data.py

@@ -16,6 +16,7 @@
     DataMismatchError,
 )
 from abc import ABC
+import pandas as pd
 
 
 class AbstractData(ABC):
@@ -26,6 +27,19 @@ def __init__(self, spec: dict, name="input_data"):
         self.name = name
         self.load_transform_ingest_data(spec)
 
+    def get_raw_data_by_cat(self, category):
+        mapping = self._data_transformer.get_target_category_columns_map()
+        # For given category, mapping gives the target_category_columns and it's values.
+        # condition filters raw_data based on the values of target_category_columns for the given category
+        condition = pd.Series(True, index=self.raw_data.index)
+        if category in mapping:
+            for col, val in mapping[category].items():
+                condition &= (self.raw_data[col] == val)
+        data_by_cat = self.raw_data[condition].reset_index(drop=True)
+        data_by_cat = self._data_transformer._format_datetime_col(data_by_cat)
+        return data_by_cat
+
+
     def get_dict_by_series(self):
         if not self._data_dict:
             for s_id in self.list_series_ids():
@@ -73,8 +87,8 @@ def _transform_data(self, spec, raw_data, **kwargs):
         return data
 
     def load_transform_ingest_data(self, spec):
-        raw_data = self._load_data(getattr(spec, self.name))
-        self.data = self._transform_data(spec, raw_data)
+        self.raw_data = self._load_data(getattr(spec, self.name))
+        self.data = self._transform_data(spec, self.raw_data)
         self._ingest_data(spec)
 
     def _ingest_data(self, spec):

ads/opctl/operator/lowcode/common/transformations.py

@@ -84,13 +84,19 @@ def _remove_trailing_whitespace(self, df):
         return df.apply(lambda x: x.str.strip() if x.dtype == "object" else x)
 
     def _set_series_id_column(self, df):
+        self._target_category_columns_map = dict()
         if not self.target_category_columns:
             df[DataColumns.Series] = "Series 1"
             self.has_artificial_series = True
         else:
             df[DataColumns.Series] = merge_category_columns(
                 df, self.target_category_columns
             )
+            merged_values = df[DataColumns.Series].unique().tolist()
+            if self.target_category_columns:
+                for value in merged_values:
+                    self._target_category_columns_map[value] = df[df[DataColumns.Series] == value][self.target_category_columns].drop_duplicates().iloc[0].to_dict()
+
             df = df.drop(self.target_category_columns, axis=1)
         return df
 
@@ -195,3 +201,25 @@ def _check_historical_dataset(self, df):
             raise DataMismatchError(
                 f"Expected {self.name} to have columns: {expected_names}, but instead found column names: {df.columns}. Is the {self.name} path correct?"
             )
+
+    """
+        Map between merged target category column values and target category column and its value
+        If target category columns are PPG_Code, Class, Num
+        Merged target category column values are Product Category 1__A__1, Product Category 2__A__2
+        Then target_category_columns_map would be
+        {
+            "Product Category 1__A__1": {
+                "PPG_Code": "Product Category 1",
+                "Class": "A",
+                "Num": 1
+            },
+             "Product Category 2__A__2": {
+                "PPG_Code": "Product Category 2",
+                "Class": "A",
+                "Num": 2
+            },
+            
+        }
+    """
+    def get_target_category_columns_map(self):
+        return self._target_category_columns_map

docs/source/release_notes.rst

@@ -2,6 +2,13 @@
 Release Notes
 =============
 
+2.11.7
+------
+Release date: April 8, 2024
+
+* Fixed bugs and introduced enhancements following our recent release, which included internal adjustments for future features and updates for the Jupyter Lab 3 upgrade.
+
+
 2.11.6
 ------
 Release date: April 3, 2024

pyproject.toml

@@ -21,7 +21,7 @@ build-backend = "flit_core.buildapi"
 
 # Required
 name = "oracle_ads" # the install (PyPI) name; name for local build in [tool.flit.module] section below
-version = "2.11.6"
+version = "2.11.7"
 
 # Optional
 description = "Oracle Accelerated Data Science SDK"

tests/unitary/with_extras/aqua/test_finetuning_handler.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from unittest import TestCase
+from unittest.mock import MagicMock
+from mock import patch
+
+from notebook.base.handlers import IPythonHandler
+from ads.aqua.extension.finetune_handler import AquaFineTuneHandler
+from ads.aqua.finetune import AquaFineTuningApp, CreateFineTuningDetails
+
+
+class TestDataset:
+    mock_valid_input = dict(
+        ft_source_id="ocid1.datasciencemodel.oc1.iad.<OCID>",
+        ft_name="test_ft_name",
+        dataset_path="oci://ds_bucket@namespace/prefix/dataset.jsonl",
+        report_path="oci://report_bucket@namespace/prefix/",
+        ft_parameters={
+            "epochs":1,
+            "learning_rate":0.02
+        },
+        shape_name="VM.GPU.A10.1",
+        replica=1,
+        validation_set_size=0.2,
+        block_storage_size=1,
+        experiment_name="test_experiment_name",
+    )
+
+    mock_finetuning_config = {
+        "shape": {
+            "VM.GPU.A10.1": {
+                "batch_size": 1,
+                "replica": 1
+            },
+        }
+    }
+
+
+class FineTuningHandlerTestCase(TestCase):
+
+    @patch.object(IPythonHandler, "__init__")
+    def setUp(self, ipython_init_mock) -> None:
+        ipython_init_mock.return_value = None
+        self.test_instance = AquaFineTuneHandler(MagicMock(), MagicMock())
+        self.test_instance.request = MagicMock()
+        self.test_instance.finish = MagicMock()
+
+    @patch.object(AquaFineTuneHandler, "get_finetuning_config")
+    @patch("ads.aqua.extension.finetune_handler.urlparse")
+    def test_get(self, mock_urlparse, mock_get_finetuning_config):
+        request_path = MagicMock(path="aqua/finetuning/config")
+        mock_urlparse.return_value = request_path
+
+        mock_get_finetuning_config.return_value = TestDataset.mock_finetuning_config
+
+        fineruning_config = self.test_instance.get(id="test_model_id")
+        mock_urlparse.assert_called()
+        mock_get_finetuning_config.assert_called_with("test_model_id")
+        assert fineruning_config == TestDataset.mock_finetuning_config
+
+    @patch.object(AquaFineTuningApp, "create")
+    def test_post(self, mock_create):
+        self.test_instance.get_json_body = MagicMock(
+            return_value = TestDataset.mock_valid_input
+        )
+        self.test_instance.post()
+
+        self.test_instance.finish.assert_called_with(
+            mock_create.return_value
+        )
+        mock_create.assert_called_with(
+            CreateFineTuningDetails(**TestDataset.mock_valid_input)
+        )
+
+    @patch.object(AquaFineTuningApp, "get_finetuning_config")
+    def test_get_finetuning_config(self, mock_get_finetuning_config):
+        mock_get_finetuning_config.return_value = TestDataset.mock_finetuning_config
+
+        self.test_instance.get_finetuning_config(model_id="test_model_id")
+
+        self.test_instance.finish.assert_called_with(
+            TestDataset.mock_finetuning_config
+        )
+        mock_get_finetuning_config.assert_called_with(
+            model_id="test_model_id"
+        )

tests/unitary/with_extras/aqua/test_model_handler.py

@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2024 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from unittest import TestCase
+from unittest.mock import MagicMock
+
+from mock import patch
+from notebook.base.handlers import IPythonHandler
+
+from ads.aqua.extension.model_handler import AquaModelHandler, AquaModelLicenseHandler
+from ads.aqua.model import AquaModelApp
+
+
+class ModelHandlerTestCase(TestCase):
+    
+    @patch.object(IPythonHandler, "__init__")
+    def setUp(self, ipython_init_mock) -> None:
+        ipython_init_mock.return_value = None
+        self.model_handler = AquaModelHandler(MagicMock(), MagicMock())
+        self.model_handler.request = MagicMock()
+        self.model_handler.finish = MagicMock()
+
+    @patch.object(AquaModelHandler, "list")
+    def test_get_no_id(self, mock_list):
+        self.model_handler.get()
+        mock_list.assert_called()
+
+    @patch.object(AquaModelHandler, "read")
+    def test_get_with_id(self, mock_read):
+        self.model_handler.get(model_id="test_model_id")
+        mock_read.assert_called_with("test_model_id")
+
+    @patch.object(AquaModelApp, "get")
+    def test_read(self, mock_get):
+        self.model_handler.read(model_id="test_model_id")
+        self.model_handler.finish.assert_called_with(
+            mock_get.return_value
+        )
+        mock_get.assert_called_with("test_model_id")
+
+    @patch.object(AquaModelApp, "clear_model_list_cache")
+    @patch("ads.aqua.extension.model_handler.urlparse")
+    def test_delete(self, mock_urlparse, mock_clear_model_list_cache):
+        request_path = MagicMock(path="aqua/model/cache")
+        mock_urlparse.return_value = request_path
+
+        self.model_handler.delete()
+        self.model_handler.finish.assert_called_with(
+            mock_clear_model_list_cache.return_value
+        )
+
+        mock_urlparse.assert_called()
+        mock_clear_model_list_cache.assert_called()
+
+    @patch.object(AquaModelApp, "list")
+    def test_list(self, mock_list):
+        self.model_handler.list()
+
+        self.model_handler.finish.assert_called_with(
+            mock_list.return_value
+        )
+        mock_list.assert_called_with(None, None)
+
+class ModelLicenseHandlerTestCase(TestCase):
+    
+    @patch.object(IPythonHandler, "__init__")
+    def setUp(self, ipython_init_mock) -> None:
+        ipython_init_mock.return_value = None
+        self.model_license_handler = AquaModelLicenseHandler(MagicMock(), MagicMock())
+        self.model_license_handler.finish = MagicMock()
+
+    @patch.object(AquaModelApp, "load_license")
+    def test_get(self, mock_load_license):
+        self.model_license_handler.get(model_id="test_model_id")
+
+        self.model_license_handler.finish.assert_called_with(
+            mock_load_license.return_value
+        )
+        mock_load_license.assert_called_with("test_model_id")