backmerged with latest

Author: yogesh266

ads/feature_store/dataset.py

@@ -500,6 +500,7 @@ def with_model_details(self, model_details: ModelDetails) -> "Dataset":
                 "The argument `model_details` has to be of type `ModelDetails`"
                 "but is of type: `{}`".format(type(model_details))
             )
+
         return self.set_spec(self.CONST_MODEL_DETAILS, model_details.to_dict())
 
     @property
@@ -549,9 +550,20 @@ def add_models(self, model_details: ModelDetails) -> "Dataset":
         if existing_model_details and existing_model_details.items:
             items = existing_model_details["items"]
             for item in items:
-                model_details.items.append(item)
+                if item not in model_details.items:
+                    model_details.items.append(item)
         self.with_model_details(model_details)
-        return self.update()
+        try:
+            return self.update()
+        except Exception as ex:
+            logger.error(
+                f"Dataset update Failed with : {type(ex)} with error message: {ex}"
+            )
+            if existing_model_details:
+                self.with_model_details(ModelDetails().with_items(existing_model_details["items"]))
+            else:
+                self.with_model_details(ModelDetails().with_items([]))
+                return self
 
     def remove_models(self, model_details: ModelDetails) -> "Dataset":
         """remove model details from the dataset, remove from the existing dataset model id list

ads/feature_store/docs/source/dataset.rst

@@ -306,7 +306,7 @@ Use the ``show()`` method on the ``Dataset`` instance to visualize the lineage o
 
 The ``show()`` method takes the following optional parameter:
 
-  - ``rankdir: (str, optional)``. Defaults to ``LR``. The allowed values are ``TB`` or ``LR``. This parameter is applicable only for ``graph`` mode and it renders the direction of the graph as either top to bottom (TB) or left to right (LR).
+- ``rankdir: (str, optional)``. Defaults to ``LR``. The allowed values are ``TB`` or ``LR``. This parameter is applicable only for ``graph`` mode and it renders the direction of the graph as either top to bottom (TB) or left to right (LR).
 
 
 .. code-block:: python3
@@ -317,3 +317,16 @@ Below is an example of the output.
 
 .. figure:: figures/dataset_lineage.png
   :width: 400
+
+
+Add Model Details
+=================
+
+You can call the ``add_models()`` method of the Dataset instance to add model ids to dataset.
+The ``.add_models()`` method takes the following parameter:
+
+- ``model_details: ModelDetails``.  ModelDetails takes ``items: List[str]`` as parameter and model ids to be passed as items.
+
+.. code-block:: python3
+
+  dataset.add_models(ModelDetails().with_items([<ocid1.datasciencemodel..<unique_id>]))

ads/feature_store/docs/source/notebook.rst

@@ -24,3 +24,8 @@ Notebook Examples
       - `Big data operations with feature store <https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/notebook/feature-store-big-data-ingestion-and-querying.ipynb>`__
       - | 1. Ingestion of data using spark magic
         | 2. Querying and exploration of data using spark magic
+
+    * - `Schema enforcement and schema evolution <https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/notebook/feature_store_flights_schema_evolution.html>`__
+      - `Schema enforcement and schema evolution <https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/notebook/feature_store_flights_schema_evolution.ipynb>`__
+      - | 1. Schema evolution is a feature that allows users to easily change a table's current schema to accommodate data that is changing over time.
+        | 2. Schema enforcement, also known as schema validation, is a safeguard in Delta Lake that ensures data quality by rejecting writes to a table that do not match the table's schema.

ads/feature_store/docs/source/quickstart.rst

@@ -55,8 +55,9 @@ Background reading to understand the concepts of Feature Store and OCI Data Scie
     compartment_id = "ocid1.compartment.<unique_id>"
     metastore_id = "ocid1.datacatalogmetastore.oc1.iad.<unique_id>"
     api_gateway_endpoint = "https://**.{region}.oci.customer-oci.com/20230101"
+    os.environ["OCI_FS_SERVICE_ENDPOINT"] = api_gateway_endpoint
 
-    ads.set_auth(auth="user_principal", client_kwargs={"service_endpoint": api_gateway_endpoint})
+    ads.set_auth(auth="api_key")
 
     # step1: Create feature store
     feature_store_resource = (

ads/feature_store/feature_group.py

@@ -323,7 +323,7 @@ def with_primary_keys(self, primary_keys: List[str]) -> "FeatureGroup":
             self.CONST_PRIMARY_KEYS,
             {
                 self.CONST_ITEMS: [
-                    {self.CONST_NAME: primary_key} for primary_key in primary_keys
+                    {self.CONST_NAME: primary_key} for primary_key in primary_keys or []
                 ]
             },
         )

ads/feature_store/mixin/oci_feature_store.py

@@ -6,13 +6,19 @@
 
 from ads.common.oci_mixin import OCIModelMixin
 import oci.feature_store
+import os
 
 
 class OCIFeatureStoreMixin(OCIModelMixin):
     @classmethod
     def init_client(
         cls, **kwargs
     ) -> oci.feature_store.feature_store_client.FeatureStoreClient:
+        # TODO: Getting the endpoint from authorizer
+        fs_service_endpoint = os.environ.get("OCI_FS_SERVICE_ENDPOINT")
+        if fs_service_endpoint:
+            kwargs = {"service_endpoint": fs_service_endpoint}
+
         client = cls._init_client(
             client=oci.feature_store.feature_store_client.FeatureStoreClient, **kwargs
         )

ads/model/generic_model.py

@@ -34,6 +34,7 @@
 from ads.evaluations import EvaluatorMixin
 from ads.feature_engineering import ADSImage
 from ads.feature_engineering.schema import Schema
+from ads.feature_store.model_details import ModelDetails
 from ads.model.artifact import ModelArtifact
 from ads.model.common.utils import (
     _extract_locals,
@@ -65,7 +66,7 @@
     Framework,
     ModelCustomMetadata,
     ModelProvenanceMetadata,
-    ModelTaxonomyMetadata,
+    ModelTaxonomyMetadata, MetadataCustomCategory,
 )
 from ads.model.model_metadata_mixin import MetadataMixin
 from ads.model.model_properties import ModelProperties
@@ -1825,6 +1826,7 @@ def save(
         remove_existing_artifact: Optional[bool] = True,
         model_version_set: Optional[Union[str, ModelVersionSet]] = None,
         version_label: Optional[str] = None,
+        featurestore_dataset=None,
         **kwargs,
     ) -> str:
         """Saves model artifacts to the model catalog.
@@ -1856,6 +1858,8 @@ def save(
             The model version set OCID, or model version set name, or `ModelVersionSet` instance.
         version_label: (str, optional). Defaults to None.
             The model version lebel.
+        featurestore_dataset: (Dataset, optional).
+            The feature store dataset
         kwargs:
             project_id: (str, optional).
                 Project OCID. If not specified, the value will be taken either
@@ -1937,6 +1941,15 @@ def save(
         # variables in case of saving model in context of model version set.
         model_version_set_id = _extract_model_version_set_id(model_version_set)
 
+        if featurestore_dataset:
+            dataset_details = {
+                "dataset-id": featurestore_dataset.id,
+                "dataset-name": featurestore_dataset.name
+            }
+            self.metadata_custom.add("featurestore.dataset", value=str(dataset_details),
+                                     category=MetadataCustomCategory.TRAINING_AND_VALIDATION_DATASETS,
+                                     description="feature store dataset", replace=True)
+
         self.dsc_model = (
             self.dsc_model.with_compartment_id(self.properties.compartment_id)
             .with_project_id(self.properties.project_id)
@@ -1965,6 +1978,10 @@ def save(
             .with_infrastructure(ModelDeploymentInfrastructure())
             .with_runtime(ModelDeploymentContainerRuntime())
         )
+        # Add the model id to the feature store dataset
+        if featurestore_dataset:
+            model_details = ModelDetails().with_items([self.model_id])
+            featurestore_dataset.add_models(model_details)
 
         return self.model_id
 

tests/integration/feature_store/test_base.py

@@ -19,6 +19,7 @@
 from ads.feature_store.input_feature_detail import FeatureDetail
 from ads.feature_store.statistics_config import StatisticsConfig
 
+
 client_kwargs = dict(
     retry_strategy=oci.retry.NoneRetryStrategy,
     service_endpoint=os.getenv("service_endpoint"),
@@ -48,7 +49,7 @@ def transformation_with_kwargs(data_frame, **kwargs):
 
 class FeatureStoreTestCase:
     # networks compartment in feature store
-    TIME_NOW = str.format("{}_{}", datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S"), int(random() * 1000))
+    TIME_NOW = str.format("{}_{}",datetime.utcnow().strftime("%Y_%m_%d_%H_%M_%S"),int(random()*1000))
     TENANCY_ID = "ocid1.tenancy.oc1..aaaaaaaa462hfhplpx652b32ix62xrdijppq2c7okwcqjlgrbknhgtj2kofa"
     COMPARTMENT_ID = "ocid1.tenancy.oc1..aaaaaaaa462hfhplpx652b32ix62xrdijppq2c7okwcqjlgrbknhgtj2kofa"
     METASTORE_ID = "ocid1.datacatalogmetastore.oc1.iad.amaaaaaabiudgxyap7tizm4gscwz7amu7dixz7ml3mtesqzzwwg3urvvdgua"
@@ -378,11 +379,10 @@ def create_transformation_resource(self, feature_store) -> "Transformation":
         transformation = feature_store.create_transformation(source_code_func=transformation_with_kwargs,
                                                              display_name="transformation_with_kwargs",
                                                              transformation_mode=TransformationMode.PANDAS)
-
         return transformation
 
     def define_feature_group_resource(
-            self, entity_id, feature_store_id
+        self, entity_id, feature_store_id
     ) -> "FeatureGroup":
         feature_group_resource = (
             FeatureGroup()
@@ -400,7 +400,7 @@ def define_feature_group_resource(
         return feature_group_resource
 
     def define_dataset_resource(
-            self, entity_id, feature_store_id, feature_group_name
+        self, entity_id, feature_store_id, feature_group_name
     ) -> "Dataset":
         name = self.get_name("petals_ds")
         dataset_resource = (
@@ -470,6 +470,7 @@ def clean_up_feature_group(feature_group):
         except Exception as ex:
             print("Failed to delete feature group: ", str(ex))
             exit(1)
+
     @staticmethod
     def clean_up_transformation(transformation):
         try:

tests/integration/feature_store/test_dataset_validations.py

@@ -91,9 +91,8 @@ def test_dataset_model_details(self):
         assert dataset.oci_dataset.id
 
         dataset.materialise()
-        updated_dataset = dataset.add_models(ModelDetails().with_items(["model_ocid"]))
-        updated_dataset.show()
-        assert updated_dataset.model_details is not None
+        dataset.add_models(ModelDetails().with_items(["model_ocid_invalid"]))
+        assert len(dataset.model_details.get("items")) == 0
         self.clean_up_dataset(dataset)
         self.clean_up_feature_group(fg)
         self.clean_up_entity(entity)