added terraform version in the command (#249)

Author: KshitizLohia

ads/feature_store/common/spark_session_singleton.py

@@ -75,8 +75,9 @@ def __init__(self, metastore_id: str = None):
                 "spark.hadoop.oracle.dcat.metastore.id", metastore_id
             ).config(
                 "spark.sql.warehouse.dir", metastore.default_managed_table_location
-            )\
-                .config("spark.driver.memory", "16G")
+            ).config(
+                "spark.driver.memory", "16G"
+            )
 
         if developer_enabled():
             # Configure spark session with delta jars only in developer mode. In other cases,

ads/feature_store/common/utils/feature_schema_mapper.py

@@ -241,7 +241,6 @@ def map_feature_type_to_pandas(feature_type):
         raise TypeError(f"Feature Type {feature_type} is not supported for pandas")
 
 
-
 def map_spark_type_to_stats_data_type(spark_type):
     """Maps the spark data types to MLM library data types
     args:

ads/feature_store/common/utils/transformation_query_validator.py

@@ -154,8 +154,10 @@ def get_features(
     return features
 
 
-def get_schema_from_pandas_df(df: pd.DataFrame):
-    spark = SparkSessionSingleton().get_spark_session()
+def get_schema_from_pandas_df(df: pd.DataFrame, feature_store_id: str):
+    spark = SparkSessionSingleton(
+        get_metastore_id(feature_store_id)
+    ).get_spark_session()
     converted_df = spark.createDataFrame(df)
     return get_schema_from_spark_df(converted_df)
 
@@ -174,27 +176,29 @@ def get_schema_from_spark_df(df: DataFrame):
     return schema_details
 
 
-def get_schema_from_df(data_frame: Union[DataFrame, pd.DataFrame]) -> List[dict]:
+def get_schema_from_df(
+    data_frame: Union[DataFrame, pd.DataFrame], feature_store_id: str
+) -> List[dict]:
     """
     Given a DataFrame, returns a list of dictionaries that describe its schema.
     If the DataFrame is a pandas DataFrame, it uses pandas methods to get the schema.
     If it's a PySpark DataFrame, it uses PySpark methods to get the schema.
     """
     if isinstance(data_frame, pd.DataFrame):
-        return get_schema_from_pandas_df(data_frame)
+        return get_schema_from_pandas_df(data_frame, feature_store_id)
     else:
         return get_schema_from_spark_df(data_frame)
 
 
 def get_input_features_from_df(
-    data_frame: Union[DataFrame, pd.DataFrame]
+    data_frame: Union[DataFrame, pd.DataFrame], feature_store_id: str
 ) -> List[FeatureDetail]:
     """
     Given a DataFrame, returns a list of FeatureDetail objects that represent its input features.
     Each FeatureDetail object contains information about a single input feature, such as its name, data type, and
     whether it's categorical or numerical.
     """
-    schema_details = get_schema_from_df(data_frame)
+    schema_details = get_schema_from_df(data_frame, feature_store_id)
     feature_details = []
 
     for schema_detail in schema_details:
@@ -263,7 +267,7 @@ def largest_matching_subset_of_primary_keys(left_feature_group, right_feature_gr
 
 
 def convert_pandas_datatype_with_schema(
-        raw_feature_details: List[dict], input_df: pd.DataFrame
+    raw_feature_details: List[dict], input_df: pd.DataFrame
 ) -> pd.DataFrame:
     feature_detail_map = {}
     columns_to_remove = []
@@ -280,21 +284,25 @@ def convert_pandas_datatype_with_schema(
                 .where(pd.notnull(input_df[column]), None)
             )
         else:
-            logger.warning("column" + column + "doesn't exist in the input feature details")
+            logger.warning(
+                "column" + column + "doesn't exist in the input feature details"
+            )
             columns_to_remove.append(column)
-    return input_df.drop(columns = columns_to_remove)
+    return input_df.drop(columns=columns_to_remove)
 
 
 def convert_spark_dataframe_with_schema(
-        raw_feature_details: List[dict], input_df: DataFrame
+    raw_feature_details: List[dict], input_df: DataFrame
 ) -> DataFrame:
     feature_detail_map = {}
     columns_to_remove = []
     for feature_details in raw_feature_details:
         feature_detail_map[feature_details.get("name")] = feature_details
     for column in input_df.columns:
         if column not in feature_detail_map.keys():
-            logger.warning("column" + column + "doesn't exist in the input feature details")
+            logger.warning(
+                "column" + column + "doesn't exist in the input feature details"
+            )
             columns_to_remove.append(column)
 
     return input_df.drop(*columns_to_remove)

ads/feature_store/common/utils/utility.py

@@ -1,11 +1,19 @@
 Quick start
 ************
+1. Create a `OCI notebook session <https://docs.oracle.com/en-us/iaas/data-science/using/create-notebook-sessions.htm>`__ to access jupyterlab interface.
 
-- Install the ``fspyspark32_p38_cpu_v1`` plugin
+2. Open the terminal in the notebook session and install the ``fspyspark32_p38_cpu_v1`` plugin
 
   ..  code-block:: shell
 
-    odsc conda install --uri https://objectstorage.us-ashburn-1.oraclecloud.com/p/b-LDcmxLYuCi3gIgvLV8KPAIih8CD2vgtBwgjEtY_CnE3HZyBHb00JGsS7OV5ls5/n/ociodscdev/b/ads_conda_pack_builds/o/PySpark_3/teamcity_20230511_061348_38940786/f227145b7ee5fc1c73a69ebaa671b81e/PySpark_3.2_and_Feature_Store.tar.gz
+    odsc conda install --uri https://objectstorage.us-ashburn-1.oraclecloud.com/n/bigdatadatasciencelarge/b/service-conda-packs-fs/o/service_pack/cpu/PySpark_3.2_and_Feature_Store/1.0/fspyspark32_p38_cpu_v1#conda
+
+3. Download the notebooks from the example notebook section.
+
+.. seealso::
+   Refer :ref:`Notebook Examples` to check out more example for using feature store
+
+4. Upload the notebook in the notebook session and run the notebook after replacing the required variables.
 
 
 Background reading to understand the concepts of Feature Store and OCI Data Science:
@@ -26,9 +34,6 @@ Background reading to understand the concepts of Feature Store and OCI Data Scie
 
    Refer `Terraform section <https://objectstorage.us-ashburn-1.oraclecloud.com/p/hh2NOgFJbVSg4amcLM3G3hkTuHyBD-8aE_iCsuZKEvIav1Wlld-3zfCawG4ycQGN/n/ociodscdev/b/oci-feature-store/o/beta/index.html#document-terraform>`__ for setting up feature store server.
 
-.. seealso::
-   Refer :ref:`Notebook Examples` to check out more example for using feature store
-
 .. warning::
 
    1. Initial implementation will not allow parallel execution of similar logical constructs. Creation will be sequential.

ads/feature_store/docs/source/figures/resource_manager.png

@@ -28,7 +28,9 @@ Release Notes
 Release notes: July 5, 2023
 
 * [FEATURE] Supporting Offline Feature Type COMPLEX
-* [[DOCS] Data Type update for Offline Feature Type COMPLEX
+* [FEATURE] Added k8 default version as v1.26.2
+* [DOCS] Data Type update for Offline Feature Type COMPLEX
+* [DOCS] Updated terraform default version as 1.1.x
 
 1.0
 ---

ads/feature_store/docs/source/figures/resource_manager_feature_store_stack.png

@@ -28,8 +28,8 @@ Feature Store users need to provide the following access permissions in order to
 
 ..  code-block:: shell
 
-    define tenancy <feature store service tenancy> as <feature store service tenancy ocid>
-    endorse group <feature store user group> to read repos in tenancy <feature store service tenancy>
+    define tenancy service_tenancy as ocid1.tenancy.oc1..aaaaaaaahzy3x4boh7ipxyft2rowu2xeglvanlfewudbnueugsieyuojkldq
+    endorse group <feature store user group> to read repos in tenancy service_tenancy
     allow group <feature store user group> to manage orm-stacks in compartment <compartmentName>
     allow group <feature store user group> to manage orm-jobs in compartment <compartmentName>
     allow group <feature store user group> to manage object-family in compartment <compartmentName>
@@ -82,46 +82,76 @@ Prerequisites
 
 1. Install `oci-cli <https://docs.oracle.com/en-us/iaas/Content/API/Concepts/cliconcepts.htm>`__ if not installed
 
-2. Download the stack from ``Terraform Stack`` column in :ref:`Release Notes`.
-
-3. (Optional: Skip if default deployment is required) To use this file just copy the example ``terraform.tfvars.example`` and save it in the outermost directory.
-
-4. (Optional: Skip if default deployment is required) Next, rename the file to ``terraform.tfvars``. You can override the example values set in this file.
-
-
 Steps
 #####
 
 .. note::
 
   Refer :ref:`Release Notes` for getting the latest conda pack and ``SERVICE_VERSION``. Remember to replace the values within angle brackets ("<>" symbols) in the command above with the relevant values for your environment. Also, Refer :ref:`User Policies` to create feature store stack for non admin users. No policies are explicitly required for admin user.
 
-
-1. Download the stack from ``Terraform Stack`` column in :ref:`Release Notes`.
-
-2. Run the shell command.
+1. Run the shell command.
   ..  code-block:: shell
 
-    oci resource-manager stack create \
-      --compartment-id <compartment-id> \
-      --config-source <path-to-downloaded-zip-file> \
-      --variables '{
-        "service_version": "<SERVICE_VERSION>",
-        "tenancy_ocid": "<TENANCY_OCID>",
-        "compartment_ocid": "<COMPARTMENT_OCID>",
-        "region": "<REGION>",
-        "user_ocid": "<USER_OCID>"
-      }' \
-      --display-name "Feature Store Stack" \
-      --working-directory "feature-store-terraform" \
-      | tee stack_output.json \
-      && stack_id=$(jq -r '.data."id"' stack_output.json) \
-      && oci resource-manager job create-apply-job \
+    rm -f feature-store-terraform.zip \
+    && wget https://objectstorage.us-ashburn-1.oraclecloud.com/p/vZogtXWwHqbkGLeqyKiqBmVxdbR4MK4nyOBqDsJNVE4sHGUY5KFi4T3mOFGA3FOy/n/idogsu2ylimg/b/oci-feature-store/o/beta/terraform/feature-store-terraform.zip \
+    && oci resource-manager stack create \
+        --compartment-id <compartment-id> \
+        --config-source <path-to-downloaded-zip-file> \
+        --terraform-version 1.1.x \
+        --variables '{
+            "service_version": "<SERVICE_VERSION>",
+            "tenancy_ocid": "<TENANCY_OCID>",
+            "compartment_ocid": "<COMPARTMENT_OCID>",
+            "region": "<REGION>",
+            "user_ocid": "<USER_OCID>"
+        }' \
+        --display-name "Feature Store Stack" \
+        --working-directory "feature-store-terraform" \
+    | tee stack_output.json \
+    && stack_id=$(jq -r '.data."id"' stack_output.json) \
+    && oci resource-manager job create-apply-job \
         --execution-plan-strategy AUTO_APPROVED \
         --stack-id $stack_id \
         --wait-for-state SUCCEEDED \
         --wait-for-state FAILED
 
+Update Feature Store Stack with the Latest using OCI CLI
+==============================
+
+Prerequisites
+#############
+
+1. Install `oci-cli <https://docs.oracle.com/en-us/iaas/Content/API/Concepts/cliconcepts.htm>`__ if not installed
+2. In order to update the stack, get the <STACK_ID> from console by navigating to `OCI Resource Manager <https://www.oracle.com/devops/resource-manager/>`__.
+
+.. figure:: figures/resource_manager.png
+
+.. figure:: figures/resource_manager_home.png
+
+.. figure:: figures/resource_manager_feature_store_stack.png
+
+
+Steps
+#####
+
+.. note::
+
+  Refer :ref:`Release Notes` for getting the latest conda pack and ``SERVICE_VERSION``. Remember to replace the values within angle brackets ("<>" symbols) in the command above with the relevant values for your environment. Also, Refer :ref:`User Policies` to create feature store stack for non admin users. No policies are explicitly required for admin user.
+
+1. Run the shell command.
+  ..  code-block:: shell
+
+    rm -f feature-store-terraform.zip \
+    && wget https://objectstorage.us-ashburn-1.oraclecloud.com/p/vZogtXWwHqbkGLeqyKiqBmVxdbR4MK4nyOBqDsJNVE4sHGUY5KFi4T3mOFGA3FOy/n/idogsu2ylimg/b/oci-feature-store/o/beta/terraform/feature-store-terraform.zip \
+    && oci resource-manager stack update \
+        --stack-id <STACK_ID> \
+        --variables '{"service_version": "<SERVICE_VERSION>", "tenancy_ocid": "<TENANCY_OCID>", "compartment_ocid": "<COMPARTMENT_OCID>", "region": "<REGION>", "user_ocid": "<USER_OCID>"}' \
+        --config-source "feature-store-terraform.zip" \
+        --working-directory "feature-store-terraform" --force \
+    && oci resource-manager job create-apply-job \
+        --execution-plan-strategy AUTO_APPROVED \
+        --stack-id <STACK_ID> \
+        --wait-for-state SUCCEEDED --wait-for-state FAILED
 
 Terraform Variables (Advanced)
 ===============================