Merge pull request #2393 from solliancenet/cj-legacy-vectorization-updates

(0.9.7-beta135.1) Updates to legacy vectorization

Author: ciprianjichici

.vscode/launch.json

@@ -84,6 +84,16 @@
             "args": ["main:app","--reload", "--port", "8000"],
             "python": "${workspaceFolder}/src/python/CodeSessionAPI/.venv/Scripts/python.exe"
         },
+        {
+            "name": "Python: Vectorization Pipeline",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "cwd" : "${workspaceFolder}/samples/vectorization-pipeline",
+            "python": "${workspaceFolder}/samples/vectorization-pipeline/.venv/Scripts/python.exe",
+            "justMyCode": true
+        },
         {
             "name": "Python: Current File",
             "type": "debugpy",

samples/vectorization-pipeline/.env.example

@@ -0,0 +1,4 @@
+# This file is an example of the .env file that should be created in the same directory as this file.
+MANAGEMENT_API_SCOPE = "MANAGEMENT_API_CLIENT_ID/.default" # Replace MANAGEMENT_API_CLIENT_ID with the actual client ID of the Management API application.
+MANAGEMENT_API_ENDPOINT = "https://..." # Replace with the actual endpoint of the Management API application.
+FOUNDATIONALLM_INSTANCE_ID = "FOUNDATIONALLM_INSTANCE_ID" # Replace FOUNDATIONALLM_INSTANCE_ID with the actual FoundationaLLM instance identifier.

samples/vectorization-pipeline/.gitignore

@@ -0,0 +1 @@
+.env

samples/vectorization-pipeline/clients/__init__.py

@@ -0,0 +1 @@
+from .management_client import ManagementClient

samples/vectorization-pipeline/clients/management_client.py

@@ -0,0 +1,150 @@
+'''
+Management API client for FoundationLLM.
+This client is used to interact with the FoundationLLM Management API.
+'''
+
+import requests
+from simple_jwt import jwt
+
+from azure.identity import AzureCliCredential
+
+
+class ManagementClient:
+    '''
+    Management API client for FoundationLLM.
+    This client is used to interact with the FoundationLLM Management API.
+    '''
+
+    def __init__(
+        self,
+        management_api_scope: str,
+        management_api_endpoint: str,
+        foundationallm_instance_id: str):
+
+        self.__management_api_scope = management_api_scope
+        self.__management_api_endpoint = management_api_endpoint
+        self.__foundationallm_instance_id = foundationallm_instance_id
+
+        self.__credential = AzureCliCredential()
+        self.__token = self.__credential.get_token(self.__management_api_scope).token
+
+        self.default_text_partitioning_profile = "text_partition_default"
+        self.default_text_embedding_profile = "text_embedding_profile_gateway_embedding3large"
+
+    def __ensure_valid_token(self):
+        '''
+        Ensure that the token is valid.
+        If the token is expired, get a new one.
+        '''
+        if not self.__token:
+            self.__token = self.__credential.get_token(self.__management_api_scope).token
+
+        if jwt.is_expired(self.__token):
+            self.__token = self.__credential.get_token(self.__management_api_scope).token
+
+    def __post_request(self, route:str, data: dict) -> dict:
+        '''
+        Post a request to the Management API.
+        :param data: The data to post.
+        :param route: The route to post to.
+        :return: The response from the API.
+        '''
+        self.__ensure_valid_token()
+
+        headers = {
+            "Authorization": f"Bearer {self.__token}",
+            "Content-Type": "application/json"
+        }
+
+        response = requests.post(
+            f"{self.__management_api_endpoint}/instances/{self.__foundationallm_instance_id}/{route}",
+            headers=headers,
+            json=data,
+            timeout=60
+        )
+
+        if response.status_code != 200:
+            raise Exception(f"Error posting to Management API: {response.text}")
+
+        return response.json()
+
+    def __get_request(self, route:str) -> dict:
+        '''
+        Post a request to the Management API.
+        :param data: The data to post.
+        :param route: The route to post to.
+        :return: The response from the API.
+        '''
+        self.__ensure_valid_token()
+
+        headers = {
+            "Authorization": f"Bearer {self.__token}"
+        }
+
+        response = requests.get(
+            f"{self.__management_api_endpoint}/instances/{self.__foundationallm_instance_id}/{route}",
+            headers=headers,
+            timeout=60
+        )
+
+        if response.status_code != 200:
+            raise Exception(f"Error posting to Management API: {response.text}")
+
+        return response.json()
+
+    def create_vectorization_pipeline(
+        self,
+        pipeline_name: str,
+        pipeline_description: str,
+        data_source_name: str,
+        vector_store_name: str
+    ) -> dict:
+ 
+        result = self.__post_request(
+            f"providers/FoundationaLLM.Vectorization/vectorizationPipelines/{pipeline_name}",
+            {
+                "name": pipeline_name,
+                "description": pipeline_description,
+                "data_source_object_id": f"/instances/{self.__foundationallm_instance_id}/providers/FoundationaLLM.DataSource/dataSources/{data_source_name}",
+                "text_partitioning_profile_object_id": f"/instances/{self.__foundationallm_instance_id}/providers/FoundationaLLM.Vectorization/textPartitioningProfiles/{self.default_text_partitioning_profile}",
+                "text_embedding_profile_object_id": f"/instances/{self.__foundationallm_instance_id}/providers/FoundationaLLM.Vectorization/textEmbeddingProfiles/{self.default_text_embedding_profile}",
+                "indexing_profile_object_id": f"/instances/{self.__foundationallm_instance_id}/providers/FoundationaLLM.Vectorization/indexingProfiles/{vector_store_name}",
+            }
+        )
+
+        return result
+
+    def get_vectorization_pipeline(
+        self,
+        pipeline_name: str
+    ) -> dict:
+
+        result = self.__get_request(
+            f"providers/FoundationaLLM.Vectorization/vectorizationPipelines/{pipeline_name}"
+        )
+
+        return result
+    
+    def activate_vectorization_pipeline(
+        self,
+        pipeline_name: str
+    ) -> dict:
+
+        result = self.__post_request(
+            f"providers/FoundationaLLM.Vectorization/vectorizationPipelines/{pipeline_name}/activate",
+            {}
+        )
+
+        return result
+
+    def get_vectorization_pipeline_execution(
+        self,
+        pipeline_name: str,
+        execution_id: str
+    ) -> dict:
+
+        result = self.__get_request(
+            f"providers/FoundationaLLM.Vectorization/vectorizationPipelines/{pipeline_name}/vectorizationPipelineExecutions/{execution_id}"
+        )
+
+        return result

samples/vectorization-pipeline/create_vectorization_pipeline.py

@@ -0,0 +1,29 @@
+'''
+This sample shows how to configure a vectorization pipeline using the FoundationaLLM Management API.
+'''
+
+import os
+from dotenv import load_dotenv
+
+from clients import ManagementClient
+
+# Load environment variables from .env file
+load_dotenv()
+
+management_client = ManagementClient(
+    os.getenv("MANAGEMENT_API_SCOPE"),
+    os.getenv("MANAGEMENT_API_ENDPOINT"),
+    os.getenv("FOUNDATIONALLM_INSTANCE_ID")
+)
+
+DATA_SOURCE_NAME = "default-storage"
+VECTOR_STORE_NAME = "default-index"
+
+result = management_client.create_vectorization_pipeline(
+    "Test02",
+    "Sample vectorization pipeline.",
+    DATA_SOURCE_NAME,
+    VECTOR_STORE_NAME
+)
+
+print(result)

samples/vectorization-pipeline/requirements.txt

@@ -0,0 +1,3 @@
+azure-identity==1.21.0
+python-dotenv==1.1.0
+simple-jwt-decode==0.10.0

samples/vectorization-pipeline/run_vectorization_pipeline.py

@@ -0,0 +1,35 @@
+'''
+This sample shows how to configure a vectorization pipeline using the FoundationaLLM Management API.
+'''
+
+import os
+from dotenv import load_dotenv
+
+from clients import ManagementClient
+
+# Load environment variables from .env file
+load_dotenv()
+
+management_client = ManagementClient(
+    os.getenv("MANAGEMENT_API_SCOPE"),
+    os.getenv("MANAGEMENT_API_ENDPOINT"),
+    os.getenv("FOUNDATIONALLM_INSTANCE_ID")
+)
+
+VECTORIZATION_PIPELINE_NAME = "Test02"
+
+result = management_client.activate_vectorization_pipeline(
+    VECTORIZATION_PIPELINE_NAME
+)
+print(result)
+
+result = management_client.get_vectorization_pipeline(
+    VECTORIZATION_PIPELINE_NAME
+)
+print(result)
+
+result = management_client.get_vectorization_pipeline_execution(
+    VECTORIZATION_PIPELINE_NAME,
+    result["resource"]["latest_execution_id"]
+)
+print(result)

src/FoundationaLLM.slnLaunch

@@ -0,0 +1,15 @@
+[
+  {
+    "Name": "Vectorization Worker and Gateway",
+    "Projects": [
+      {
+        "Path": "dotnet\\VectorizationWorker\\VectorizationWorker.csproj",
+        "Action": "Start"
+      },
+      {
+        "Path": "dotnet\\GatewayAPI\\GatewayAPI.csproj",
+        "Action": "Start"
+      }
+    ]
+  }
+]

src/dotnet/Common/Constants/ResourceProviders/VectorizationResourceProviderMetadata.cs

@@ -1,5 +1,6 @@
 using FoundationaLLM.Common.Constants.Authorization;
 using FoundationaLLM.Common.Models.ResourceProviders;
+using FoundationaLLM.Common.Models.ResourceProviders.Agent.AgentFiles;
 using FoundationaLLM.Common.Models.ResourceProviders.Vectorization;
 using FoundationaLLM.Common.Models.Vectorization;
 
@@ -36,7 +37,22 @@ public static class VectorizationResourceProviderMetadata
                         new ResourceTypeAction(ResourceProviderActions.Purge, true, false, [
                             new ResourceTypeAllowedTypes(HttpMethod.Post.Method, AuthorizableOperations.Delete, [], [], [typeof(ResourceProviderActionResult)])
                         ])
-                    ]
+                    ],
+                    SubTypes = new()
+                    {
+                        {
+                            VectorizationResourceTypeNames.VectorizationPipelineExecutions,
+                            new ResourceTypeDescriptor (
+                                VectorizationResourceTypeNames.VectorizationPipelineExecutions,
+                                typeof(VectorizationPipelineExecution))
+                            {
+                                AllowedTypes = [
+                                    new ResourceTypeAllowedTypes(HttpMethod.Get.Method, AuthorizableOperations.Write, [], [], [typeof(ResourceProviderGetResult<VectorizationPipelineExecution>)]),
+                                ],
+                                Actions = []
+                            }
+                        }
+                    }
                 }
             },
             {

src/dotnet/Common/Constants/ResourceProviders/VectorizationResourceTypeNames.cs

@@ -10,6 +10,11 @@ public static class VectorizationResourceTypeNames
         /// </summary>
         public const string VectorizationPipelines = "vectorizationPipelines";
 
+        /// <summary>
+        /// Vectorization pipeline executions.
+        /// </summary>
+        public const string VectorizationPipelineExecutions = "vectorizationPipelineExecutions";
+
         /// <summary>
         /// Vectorization requests.
         /// </summary>

src/dotnet/Common/Interfaces/IStorageService.cs

@@ -51,6 +51,21 @@ public interface IStorageService
         /// <returns></returns>
         Task WriteFileAsync(string containerName, string filePath, string fileContent, string? contentType, CancellationToken cancellationToken);
 
+        /// <summary>
+        /// Updates a specified JSON file in the storage.
+        /// </summary>
+        /// <typeparam name="T">The type of the object persisted in the JSON file.</typeparam>
+        /// <param name="containerName">The name of the container where the file is located.</param>
+        /// <param name="filePath">The path of the file to update.</param>
+        /// <param name="objectTransformer">A function that updates the object persisted in the JSON file.</param>
+        /// <param name="cancellationToken">The cancellation token that signals that operations should be cancelled.</param>
+        /// <returns></returns>
+        Task UpdateJSONFileAsync<T>(
+            string containerName,
+            string filePath,
+            Func<T, T> objectTransformer,
+            CancellationToken cancellationToken) where T : class;
+
         /// <summary>
         /// Deletes a file from storage.
         /// </summary>

src/dotnet/Common/Models/ResourceProviders/Vectorization/VectorizationRequest.cs

@@ -52,13 +52,20 @@ public class VectorizationRequest : ResourceBase
         [JsonPropertyName("pipeline_execution_id")]
         public string? PipelineExecutionId { get; set; }
 
+        /// <summary>
+        /// Gets or sets the time when the pipeline execution started.
+        /// </summary>
+        [JsonPropertyOrder(53)]
+        [JsonPropertyName("pipeline_execution_start")]
+        public DateTimeOffset? PipelineExecutionStart { get; set; }
+
         /// <summary>
         /// The <see cref="VectorizationProcessingState"/> indicating the current state of the vectorization request.
         /// </summary>
         [JsonPropertyOrder(100)]
         [JsonPropertyName("processing_state")]
         [JsonConverter(typeof(JsonStringEnumConverter))]
-        public VectorizationProcessingState ProcessingState { get; set; }
+        public VectorizationProcessingState ProcessingState { get; set; } = VectorizationProcessingState.New;
 
         /// <summary>
         /// The time when the vectorization request started being processed.
@@ -179,10 +186,27 @@ public class VectorizationRequest : ResourceBase
         public bool Expired =>
             (DateTime.UtcNow - LastSuccessfulStepTime).TotalHours > 240;
 
+        /// <summary>
+        /// Gets the date from the name of the vectorization request.
+        /// </summary>
+        /// <returns>The date parsed from the name.</returns>
+        public DateOnly GetDateFromName() => GetDate(Name);
+
         /// <summary>
         /// Set default property values.
         /// </summary>
         public VectorizationRequest() =>
             Type = "vectorization-request";
+
+        /// <summary>
+        /// Gets the date from the name of the vectorization request.
+        /// </summary>
+        /// <param name="vectorizationRequestName">The name of the vectorization request.</param>
+        /// <remarks>
+        /// The required format of the vectorization request name is "yyyyMMdd-..." (e.g., "20250502-abc").
+        /// </remarks>
+        /// <returns>The date parsed from the name.</returns>
+        public static DateOnly GetDate(string vectorizationRequestName) =>
+            DateOnly.FromDateTime(DateTime.ParseExact(vectorizationRequestName.Split('-')[0], "yyyyMMdd", null));
     }
 }

src/dotnet/Common/Models/Vectorization/VectorizationPipeline.cs

@@ -53,6 +53,12 @@ public class VectorizationPipeline : ResourceBase
         [JsonPropertyName("trigger_cron_schedule")]
         public string? TriggerCronSchedule { get; set; }
 
+        /// <summary>
+        /// Gets or sets the identifier of the latest execution of the pipeline.
+        /// </summary>
+        [JsonPropertyName("latest_execution_id")]
+        public string? LatestExecutionId { get; set; }
+
         /// <summary>
         /// Set default property values.
         /// </summary>