diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 290c5501bc07..b2a97f798413 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -59,6 +59,7 @@ + diff --git a/dotnet/samples/Concepts/Concepts.csproj b/dotnet/samples/Concepts/Concepts.csproj index 0a8add50d2d2..31d6a392de0e 100644 --- a/dotnet/samples/Concepts/Concepts.csproj +++ b/dotnet/samples/Concepts/Concepts.csproj @@ -14,6 +14,7 @@ + @@ -22,6 +23,7 @@ + diff --git a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs index 6848b38af48f..4fd62592adf3 100644 --- a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs +++ b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs @@ -40,6 +40,16 @@ public IVectorStoreRecordCollection GetCollection( return embeddingStore; } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._decoratedVectorStore.GetService(serviceType, serviceKey); + } + /// public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) { diff --git a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs index 000cb1ebba07..7ac4352e401f 100644 --- a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs +++ b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs @@ -47,6 +47,7 @@ public TextEmbeddingVectorStoreRecordCollection(IVectorStoreRecordCollection + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] public string CollectionName => this._decoratedVectorStoreRecordCollection.CollectionName; /// @@ -132,6 +133,16 @@ public async Task> VectorizableTextSearchAsync(stri return await this.VectorizedSearchAsync(embeddingValue, options, cancellationToken).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._decoratedVectorStoreRecordCollection.GetService(serviceType, serviceKey); + } + /// /// Generate and add embeddings for each embedding field that has a on the provided record. /// diff --git a/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs b/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs index 2165c96aabb2..ae72fd36c9d7 100644 --- a/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs +++ b/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs @@ -1,12 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Diagnostics; using Azure.Identity; +using Azure.Monitor.OpenTelemetry.Exporter; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AzureOpenAI; using Microsoft.SemanticKernel.Connectors.InMemory; using Microsoft.SemanticKernel.Embeddings; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; namespace Memory; @@ -14,8 +19,13 @@ namespace Memory; /// A simple example showing how to ingest data into a vector store and then use vector search to find related records to a given string /// with enabled telemetry. /// -public class VectorStore_Telemetry(ITestOutputHelper output) : BaseTest(output) +public class VectorStore_Telemetry(ITestOutputHelper output) : BaseTest(output, redirectSystemConsoleOutput: true) { + /// + /// Instance of for the example's main activity. + /// + private static readonly ActivitySource s_activitySource = new("VectorStoreTelemetry.Example"); + [Fact] public async Task LoggingManualRegistrationAsync() { @@ -91,6 +101,143 @@ public async Task LoggingDependencyInjectionAsync() // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data. } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TracingAsync(bool useApplicationInsights) + { + // Create an embedding generation service. + var textEmbeddingGenerationService = new AzureOpenAITextEmbeddingGenerationService( + TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, + TestConfiguration.AzureOpenAIEmbeddings.Endpoint, + new AzureCliCredential()); + + // Manually construct an InMemory vector store with enabled OpenTelemetry. + var vectorStore = new InMemoryVectorStore() + .AsBuilder() + .UseOpenTelemetry() + .Build(); + + using var tracerProvider = GetTracerProvider(useApplicationInsights); + + using var activity = s_activitySource.StartActivity("MainActivity"); + Console.WriteLine($"Operation/Trace ID: {Activity.Current?.TraceId}"); + + await RunExampleAsync(textEmbeddingGenerationService, vectorStore); + + // Output: + // Activity.DisplayName: create_collection_if_not_exists skglossary + // Activity.Duration: 00:00:00.0026442 + // Activity.Tags: + // db.operation.name: create_collection_if_not_exists + // db.collection.name: skglossary + // db.system.name: inmemory + // Activity.DisplayName: upsert skglossary + // Activity.Duration: 00:00:00.0010686 + // Activity.Tags: + // db.operation.name: upsert + // db.collection.name: skglossary + // db.system.name: inmemory + // and more... + + // Search string: What is an Application Programming Interface + // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data. + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MeteringAsync(bool useApplicationInsights) + { + // Create an embedding generation service. + var textEmbeddingGenerationService = new AzureOpenAITextEmbeddingGenerationService( + TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, + TestConfiguration.AzureOpenAIEmbeddings.Endpoint, + new AzureCliCredential()); + + // Manually construct an InMemory vector store with enabled OpenTelemetry. + var vectorStore = new InMemoryVectorStore() + .AsBuilder() + .UseOpenTelemetry() + .Build(); + + using var meterProvider = GetMeterProvider(useApplicationInsights); + + await RunExampleAsync(textEmbeddingGenerationService, vectorStore); + + // Output: + // Metric Name: db.client.operation.duration, Duration of database client operations., Unit: s, Meter: Experimental.Microsoft.Extensions.VectorData + + // (2025-03-25T20:03:17.9938116Z, 2025-03-25T20:03:20.2214978Z] db.collection.name: skglossary db.operation.name: create_collection_if_not_exists db.system.name: inmemory Histogram + // Value: Sum: 0.0015761 Count: 1 Min: 0.0015761 Max: 0.0015761 + + // (2025-03-25T20:03:17.9938116Z, 2025-03-25T20:03:20.2214978Z] db.collection.name: skglossary db.operation.name: upsert db.system.name: inmemory Histogram + // Value: Sum: 0.0011708 Count: 3 Min: 7E-06 Max: 0.0009944 + + // (2025-03-25T20:03:17.9938116Z, 2025-03-25T20:03:20.2214978Z] db.collection.name: skglossary db.operation.name: vectorized_search db.system.name: inmemory Histogram + // Value: Sum: 0.0109487 Count: 1 Min: 0.0109487 Max: 0.0109487 + + // Search string: What is an Application Programming Interface + // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data. + } + + #region private + + private TracerProvider? GetTracerProvider(bool useApplicationInsights) + { + var tracerProviderBuilder = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Vector Data Tracing Example")) + .AddSource("Experimental.Microsoft.Extensions.VectorData*") + .AddSource(s_activitySource.Name); + + if (useApplicationInsights) + { + var connectionString = TestConfiguration.ApplicationInsights.ConnectionString; + + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ConfigurationNotFoundException( + nameof(TestConfiguration.ApplicationInsights), + nameof(TestConfiguration.ApplicationInsights.ConnectionString)); + } + + tracerProviderBuilder.AddAzureMonitorTraceExporter(o => o.ConnectionString = connectionString); + } + else + { + tracerProviderBuilder.AddConsoleExporter(); + } + + return tracerProviderBuilder.Build(); + } + + private MeterProvider? GetMeterProvider(bool useApplicationInsights) + { + var meterProviderBuilder = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Vector Data Metering Example")) + .AddMeter("Experimental.Microsoft.Extensions.VectorData*"); + + if (useApplicationInsights) + { + var connectionString = TestConfiguration.ApplicationInsights.ConnectionString; + + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ConfigurationNotFoundException( + nameof(TestConfiguration.ApplicationInsights), + nameof(TestConfiguration.ApplicationInsights.ConnectionString)); + } + + meterProviderBuilder.AddAzureMonitorMetricExporter(o => o.ConnectionString = connectionString); + } + else + { + meterProviderBuilder.AddConsoleExporter(); + } + + return meterProviderBuilder.Build(); + } + private async Task RunExampleAsync( ITextEmbeddingGenerationService textEmbeddingGenerationService, IVectorStore vectorStore) @@ -177,4 +324,6 @@ private static IEnumerable CreateGlossaryEntries() Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)." }; } + + #endregion } diff --git a/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs b/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs index f6a3d4ab6356..c9cf9e4310ed 100644 --- a/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs +++ b/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs @@ -150,6 +150,16 @@ public async Task> VectorizableTextSearchAsync(stri return await vectorizedSearch.VectorizedSearchAsync(vectorizedQuery, options, cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + vectorizedSearch.GetService(serviceType, serviceKey); + } } /// diff --git a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs index b786c8d8fa58..a9c787aeacdf 100644 --- a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs @@ -41,6 +41,7 @@ public AzureAISearchVectorStoreRecordCollectionTests() this._searchClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock.Setup(x => x.GetSearchClient(TestCollectionName)).Returns(this._searchClientMock.Object); + this._searchIndexClientMock.Setup(x => x.ServiceName).Returns("TestService"); } [Theory] diff --git a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs index b79b048a5f38..f4eed15172fb 100644 --- a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs @@ -32,6 +32,7 @@ public AzureAISearchVectorStoreTests() this._searchClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock.Setup(x => x.GetSearchClient(TestCollectionName)).Returns(this._searchClientMock.Object); + this._searchIndexClientMock.Setup(x => x.ServiceName).Returns("TestService"); } [Fact] diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs index 5329cdf3cee4..f863bb8340c3 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Azure AI Search vector store. /// @@ -19,8 +21,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; /// public class AzureAISearchVectorStore : IVectorStore { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureAISearch"; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. private readonly SearchIndexClient _searchIndexClient; @@ -39,6 +41,12 @@ public AzureAISearchVectorStore(SearchIndexClient searchIndexClient, AzureAISear this._searchIndexClient = searchIndexClient; this._options = options ?? new AzureAISearchVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreSystemName = "azure.aisearch", + DatabaseName = searchIndexClient.ServiceName + }; } /// @@ -75,22 +83,38 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat var indexNamesEnumerable = this._searchIndexClient.GetIndexNamesAsync(cancellationToken).ConfigureAwait(false); var indexNamesEnumerator = indexNamesEnumerable.GetAsyncEnumerator(); - var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator).ConfigureAwait(false); + var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreSystemName).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.name; - nextResult = await GetNextIndexNameAsync(indexNamesEnumerator).ConfigureAwait(false); + nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreSystemName).ConfigureAwait(false); } } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(SearchIndexClient) ? this._searchIndexClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Helper method to get the next index name from the enumerator with a try catch around the move next call to convert /// any to , since try catch is not supported /// around a yield return. /// /// The enumerator to get the next result from. + /// The vector store system name. /// A value indicating whether there are more results and the current string if true. - private static async Task<(string name, bool more)> GetNextIndexNameAsync(ConfiguredCancelableAsyncEnumerable.Enumerator enumerator) + private static async Task<(string name, bool more)> GetNextIndexNameAsync( + ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, + string? vectorStoreSystemName) { const string OperationName = "GetIndexNames"; @@ -103,7 +127,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = vectorStoreSystemName, OperationName = OperationName }; } @@ -111,7 +135,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = vectorStoreSystemName, OperationName = OperationName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs index eda71258ef24..e7dd228b2446 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs @@ -18,6 +18,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Azure AI Search as the underlying storage. /// @@ -29,8 +31,8 @@ public class AzureAISearchVectorStoreRecordCollection : IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureAISearch"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -141,6 +143,13 @@ public AzureAISearchVectorStoreRecordCollection(SearchIndexClient searchIndexCli { this._mapper = new AzureAISearchGenericDataModelMapper(this._propertyReader.RecordDefinition) as IVectorStoreRecordMapper; } + + this._collectionMetadata = new() + { + VectorStoreSystemName = "azure.aisearch", + DatabaseName = searchIndexClient.ServiceName, + CollectionName = collectionName + }; } /// @@ -162,8 +171,8 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - CollectionName = this._collectionName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = "GetIndex" }; } @@ -488,6 +497,20 @@ public Task> HybridSearchAsync(TVector vec return this.SearchAndMapToDataModelAsync(keywordsCombined, searchOptions, internalOptions.IncludeVectors, cancellationToken); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(SearchIndexClient) ? this._searchIndexClient : + serviceType == typeof(SearchClient) ? this._searchClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Get the document with the given key and map it to the data model using the configured mapper type. /// @@ -517,7 +540,7 @@ public Task> HybridSearchAsync(TVector vec } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, OperationName, () => this._mapper!.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); @@ -580,7 +603,7 @@ private Task> MapToStorageModelAndUploadDocumentA if (this._mapper is not null) { var jsonObjects = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, OperationName, () => records.Select(this._mapper!.MapFromDataToStorageModel)); @@ -608,7 +631,7 @@ private async IAsyncEnumerable> MapSearchResultsAsyn await foreach (var result in results.ConfigureAwait(false)) { var document = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, operationName, () => this._options.JsonObjectCustomMapper!.MapFromStorageToDataModel(result.Document, new() { IncludeVectors = includeVectors })); @@ -688,8 +711,8 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - CollectionName = this._collectionName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = operationName }; } @@ -697,8 +720,8 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - CollectionName = this._collectionName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = operationName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs index 76dc9e8500a4..7d86b598da80 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs @@ -9,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Azure CosmosDB MongoDB vector store. /// @@ -17,6 +19,9 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; /// public class AzureCosmosDBMongoDBVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that can be used to manage the collections in Azure CosmosDB MongoDB. private readonly IMongoDatabase _mongoDatabase; @@ -34,6 +39,12 @@ public AzureCosmosDBMongoDBVectorStore(IMongoDatabase mongoDatabase, AzureCosmos this._mongoDatabase = mongoDatabase; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreSystemName = "azure.cosmosdbmongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName + }; } /// @@ -75,4 +86,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs index 13e31475447d..5ec62d911a1e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs @@ -17,6 +17,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Azure CosmosDB MongoDB as the underlying storage. /// @@ -25,8 +27,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; public class AzureCosmosDBMongoDBVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureCosmosDBMongoDB"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// Property name to be used for search similarity score value. private const string ScorePropertyName = "similarityScore"; @@ -93,6 +95,13 @@ public AzureCosmosDBMongoDBVectorStoreRecordCollection( this._vectorStoragePropertyNames = this._propertyReader.VectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); this._mapper = this.InitializeMapper(); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "azure.cosmosdbmongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, + CollectionName = collectionName + }; } /// @@ -108,8 +117,8 @@ public virtual async Task CreateCollectionAsync(CancellationToken cancellationTo { throw new VectorStoreOperationException("Collection already exists.") { - VectorStoreType = DatabaseName, - CollectionName = this.CollectionName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = "CreateCollection" }; } @@ -173,7 +182,7 @@ public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new() { IncludeVectors = includeVectors })); @@ -200,7 +209,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( if (record is not null) { yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new())); @@ -218,7 +227,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancel var replaceOptions = new ReplaceOptions { IsUpsert = true }; var storageModel = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -324,6 +333,20 @@ public virtual async Task> VectorizedSearchAsync(this.EnumerateAndMapSearchResultsAsync(cursor, searchOptions, cancellationToken)); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType == typeof(IMongoCollection) ? this._mongoCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async Task CreateIndexesAsync(string collectionName, CancellationToken cancellationToken) @@ -399,7 +422,7 @@ private async IAsyncEnumerable> EnumerateAndMapSearc { var score = response[ScorePropertyName].AsDouble; var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, new())); @@ -438,7 +461,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -455,7 +478,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs index 39320e0a8ae2..6127b3936ea6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs @@ -9,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Azure CosmosDB NoSQL vector store. /// @@ -17,6 +19,9 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; /// public class AzureCosmosDBNoSQLVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that can be used to manage the collections in Azure CosmosDB NoSQL. private readonly Database _database; @@ -34,6 +39,12 @@ public AzureCosmosDBNoSQLVectorStore(Database database, AzureCosmosDBNoSQLVector this._database = database; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreSystemName = "azure.cosmosdbnosql", + DatabaseName = database.Id + }; } /// @@ -84,4 +95,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(Database) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs index dd0e245c8004..2bb3775852f5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs @@ -17,6 +17,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Azure CosmosDB NoSQL as the underlying storage. /// @@ -28,8 +30,8 @@ public class AzureCosmosDBNoSQLVectorStoreRecordCollection : IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureCosmosDBNoSQL"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// A of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -164,6 +166,13 @@ public AzureCosmosDBNoSQLVectorStoreRecordCollection( .Concat([this._propertyReader.KeyProperty]) .Select(x => this._storagePropertyNames[x.DataModelPropertyName]) .ToList(); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "azure.cosmosdbnosql", + DatabaseName = database.Id, + CollectionName = collectionName + }; } /// @@ -442,6 +451,19 @@ public Task> HybridSearchAsync(TVector vec #endregion + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(Database) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private void VerifyVectorType(TVector? vector) @@ -468,7 +490,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -655,7 +677,7 @@ private async IAsyncEnumerable InternalGetAsync( await foreach (var jsonObject in this.GetItemsAsync(queryDefinition, cancellationToken).ConfigureAwait(false)) { yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); @@ -671,7 +693,7 @@ private async Task InternalUpsertAsync( const string OperationName = "UpsertItem"; var jsonObject = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -751,7 +773,7 @@ private async IAsyncEnumerable> MapSearchResultsAsyn jsonObject.Remove(scorePropertyName); var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs index 2db7013b0d27..c974b6aa5809 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs @@ -9,11 +9,16 @@ namespace Microsoft.SemanticKernel.Connectors.InMemory; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Service for storing and retrieving vector records, and managing vector record collections, that uses an in memory dictionary as the underlying storage. /// public sealed class InMemoryVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// Internal storage for the record collection. private readonly ConcurrentDictionary> _internalCollection; @@ -26,6 +31,11 @@ public sealed class InMemoryVectorStore : IVectorStore public InMemoryVectorStore() { this._internalCollection = new(); + + this._metadata = new() + { + VectorStoreSystemName = "inmemory" + }; } /// @@ -35,6 +45,11 @@ public InMemoryVectorStore() internal InMemoryVectorStore(ConcurrentDictionary> internalCollection) { this._internalCollection = internalCollection; + + this._metadata = new() + { + VectorStoreSystemName = "inmemory" + }; } /// @@ -59,4 +74,17 @@ public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cance { return this._internalCollection.Keys.ToAsyncEnumerable(); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(ConcurrentDictionary>) ? this._internalCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs index 6fbcdf2633bf..1effc75a45a0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs @@ -12,6 +12,8 @@ namespace Microsoft.SemanticKernel.Connectors.InMemory; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses an in memory dictionary as the underlying storage. /// @@ -22,6 +24,9 @@ public sealed class InMemoryVectorStoreRecordCollection : IVector #pragma warning restore CA1711 // Identifiers should not have incorrect suffix where TKey : notnull { + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + /// A set of types that vectors on the provided model may have. private static readonly HashSet s_supportedVectorTypes = [ @@ -81,6 +86,12 @@ public InMemoryVectorStoreRecordCollection(string collectionName, InMemoryVector // Assign resolvers. this._vectorResolver = CreateVectorResolver(this._options.VectorResolver, this._vectorProperties); this._keyResolver = CreateKeyResolver(this._options.KeyResolver, this._propertyReader.KeyProperty); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "inmemory", + CollectionName = collectionName + }; } /// @@ -278,6 +289,19 @@ public async Task> VectorizedSearchAsync(T return new VectorSearchResults(vectorSearchResultList) { TotalCount = count }; } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(ConcurrentDictionary>) ? this._internalCollections : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Get the collection dictionary from the internal storage, throws if it does not exist. /// diff --git a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs index 27169e3e9557..ee2699e9a3fb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs @@ -9,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a MongoDB vector store. /// @@ -17,6 +19,9 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; /// public class MongoDBVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that can be used to manage the collections in MongoDB. private readonly IMongoDatabase _mongoDatabase; @@ -34,6 +39,12 @@ public MongoDBVectorStore(IMongoDatabase mongoDatabase, MongoDBVectorStoreOption this._mongoDatabase = mongoDatabase; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreSystemName = "mongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName + }; } /// @@ -75,4 +86,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs index dc2aa163a803..b5c7c6822abb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs @@ -16,6 +16,8 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses MongoDB as the underlying storage. /// @@ -24,8 +26,8 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; public class MongoDBVectorStoreRecordCollection : IVectorStoreRecordCollection, IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "MongoDB"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// Property name to be used for search similarity score value. private const string ScorePropertyName = "similarityScore"; @@ -95,6 +97,13 @@ public MongoDBVectorStoreRecordCollection( this._vectorStoragePropertyNames = this._propertyReader.VectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); this._mapper = this.InitializeMapper(); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "mongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, + CollectionName = collectionName + }; } /// @@ -110,7 +119,7 @@ public virtual async Task CreateCollectionAsync(CancellationToken cancellationTo { throw new VectorStoreOperationException("Collection already exists.") { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = "CreateCollection" }; @@ -181,7 +190,7 @@ public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new() { IncludeVectors = includeVectors })); @@ -208,7 +217,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( if (record is not null) { yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new())); @@ -226,7 +235,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancel var replaceOptions = new ReplaceOptions { IsUpsert = true }; var storageModel = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -373,6 +382,20 @@ public async Task> HybridSearchAsync(TVect cancellationToken).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType == typeof(IMongoCollection) ? this._mongoCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async Task CreateIndexesAsync(string collectionName, CancellationToken cancellationToken) @@ -490,7 +513,7 @@ private async IAsyncEnumerable> EnumerateAndMapSearc { var score = response[ScorePropertyName].AsDouble; var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, new() { IncludeVectors = includeVectors })); @@ -529,7 +552,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -546,7 +569,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -577,7 +600,7 @@ private async Task RunOperationWithRetryAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -611,7 +634,7 @@ private async Task RunOperationWithRetryAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs index a072ea6e7336..138fe7b61947 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Pinecone vector store. /// @@ -18,11 +20,14 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; /// public class PineconeVectorStore : IVectorStore { - private const string DatabaseName = "Pinecone"; + private const string ListCollectionsName = "ListCollections"; private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreOptions _options; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// /// Initializes a new instance of the class. /// @@ -34,6 +39,11 @@ public PineconeVectorStore(Sdk.PineconeClient pineconeClient, PineconeVectorStor this._pineconeClient = pineconeClient; this._options = options ?? new PineconeVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreSystemName = "pinecone" + }; } /// @@ -71,8 +81,8 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - OperationName = "ListCollections" + VectorStoreType = this._metadata.VectorStoreSystemName, + OperationName = ListCollectionsName }; } @@ -84,4 +94,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(Sdk.PineconeClient) ? this._pineconeClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 3da753575141..92816386cdfc 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -13,6 +13,8 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Pinecone as the underlying storage. /// @@ -21,10 +23,11 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - private const string DatabaseName = "Pinecone"; - private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreRecordCollectionOptions _options; private readonly VectorStoreRecordPropertyReader _propertyReader; @@ -78,6 +81,12 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st // Default Mapper. this._mapper = new PineconeVectorStoreRecordMapper(this._propertyReader); } + + this._collectionMetadata = new() + { + VectorStoreSystemName = "pinecone", + CollectionName = collectionName + }; } /// @@ -153,7 +162,7 @@ public virtual async Task DeleteCollectionAsync(CancellationToken cancellationTo { throw new VectorStoreOperationException("Call to vector store failed.", other) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = "DeleteCollection" }; @@ -183,7 +192,7 @@ public virtual async Task DeleteCollectionAsync(CancellationToken cancellationTo StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true }; return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, "Get", () => this._mapper.MapFromStorageToDataModel(result, mapperOptions)); @@ -219,7 +228,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true }; var records = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, "GetBatch", () => response.Vectors.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); @@ -274,7 +283,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken Verify.NotNull(record); var vector = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, "Upsert", () => this._mapper.MapFromDataToStorageModel(record)); @@ -298,7 +307,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -372,7 +381,7 @@ public virtual async Task> VectorizedSearchAsync skippedResults.Select(x => new VectorSearchResult(this._mapper.MapFromStorageToDataModel(new Sdk.Vector() @@ -387,6 +396,19 @@ public virtual async Task> VectorizedSearchAsync + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(Sdk.PineconeClient) ? this._pineconeClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + private async Task RunIndexOperationAsync(string operationName, Func> operation) { try @@ -405,7 +427,7 @@ private async Task RunIndexOperationAsync(string operationName, Func RunCollectionOperationAsync(string operationName, Func< { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs index 020aa46dbda6..9bb3d7e617d1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs @@ -21,6 +21,11 @@ internal interface IPostgresVectorStoreDbClient /// NpgsqlDataSource DataSource { get; } + /// + /// The name of the database. + /// + string? DatabaseName { get; } + /// /// Check if a table exists. /// diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs index f8784890e83a..ef2491a13b48 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs @@ -8,9 +8,6 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres; internal static class PostgresConstants { - /// The name of this database for telemetry purposes. - public const string DatabaseName = "Postgres"; - /// A of types that a key on the provided model may have. public static readonly HashSet SupportedKeyTypes = [ diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs index 0f61e692ae7f..2f067ca132a5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs @@ -8,6 +8,8 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Represents a vector store implementation using PostgreSQL. /// @@ -17,6 +19,9 @@ public class PostgresVectorStore : IVectorStore private readonly NpgsqlDataSource? _dataSource; private readonly PostgresVectorStoreOptions _options; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// /// Initializes a new instance of the class. /// @@ -27,6 +32,12 @@ public PostgresVectorStore(NpgsqlDataSource dataSource, PostgresVectorStoreOptio this._dataSource = dataSource; this._options = options ?? new PostgresVectorStoreOptions(); this._postgresClient = new PostgresVectorStoreDbClient(this._dataSource, this._options.Schema); + + this._metadata = new() + { + VectorStoreSystemName = "postgresql", + DatabaseName = this._postgresClient.DatabaseName + }; } /// @@ -38,6 +49,12 @@ internal PostgresVectorStore(IPostgresVectorStoreDbClient postgresDbClient, Post { this._postgresClient = postgresDbClient; this._options = options ?? new PostgresVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreSystemName = "postgresql", + DatabaseName = this._postgresClient.DatabaseName + }; } /// @@ -46,7 +63,8 @@ public virtual IAsyncEnumerable ListCollectionNamesAsync(CancellationTok const string OperationName = "ListCollectionNames"; return PostgresVectorStoreUtils.WrapAsyncEnumerableAsync( this._postgresClient.GetTablesAsync(cancellationToken), - OperationName + OperationName, + vectorStoreSystemName: this._metadata.VectorStoreSystemName ); } @@ -74,4 +92,17 @@ public virtual IVectorStoreRecordCollection GetCollection ?? throw new InvalidOperationException("Failed to cast record collection."); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(NpgsqlDataSource) ? this._dataSource : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs index a167aad9cd02..9938a910b269 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs @@ -26,10 +26,14 @@ internal class PostgresVectorStoreDbClient(NpgsqlDataSource dataSource, string s { private readonly string _schema = schema; + private readonly NpgsqlConnectionStringBuilder _connectionStringBuilder = new(dataSource.ConnectionString); + private IPostgresVectorStoreCollectionSqlBuilder _sqlBuilder = new PostgresVectorStoreCollectionSqlBuilder(); public NpgsqlDataSource DataSource { get; } = dataSource; + public string? DatabaseName => this._connectionStringBuilder.Database; + /// public async Task DoesTableExistsAsync(string tableName, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs index ce619398bf99..f8a6523b5833 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Represents a collection of vector store records in a Postgres database. /// @@ -24,6 +26,9 @@ public class PostgresVectorStoreRecordCollection : IVectorStoreRe /// public string CollectionName { get; } + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + /// Postgres client that is used to interact with the database. private readonly IPostgresVectorStoreDbClient _client; @@ -102,6 +107,13 @@ internal PostgresVectorStoreRecordCollection(IPostgresVectorStoreDbClient client { this._mapper = new PostgresVectorStoreRecordMapper(this._propertyReader); } + + this._collectionMetadata = new() + { + VectorStoreSystemName = "postgresql", + DatabaseName = this._client.DatabaseName, + CollectionName = collectionName + }; } /// @@ -146,7 +158,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancella const string OperationName = "Upsert"; var storageModel = VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -173,7 +185,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -207,7 +219,7 @@ await this.RunOperationAsync(OperationName, () => if (row is null) { return default; } return VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(row, new() { IncludeVectors = includeVectors })); @@ -227,14 +239,15 @@ public virtual IAsyncEnumerable GetBatchAsync(IEnumerable keys, G this._client.GetBatchAsync(this.CollectionName, keys, this._propertyReader.RecordDefinition.Properties, includeVectors, cancellationToken) .SelectAsync(row => VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(row, new() { IncludeVectors = includeVectors })), cancellationToken ), OperationName, - this.CollectionName + this.CollectionName, + this._collectionMetadata.VectorStoreSystemName ); } @@ -303,7 +316,7 @@ public virtual Task> VectorizedSearchAsync .SelectAsync(result => { var record = VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel( @@ -317,6 +330,19 @@ public virtual Task> VectorizedSearchAsync }); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(NpgsqlDataSource) ? this._client.DataSource : + serviceType.IsInstanceOfType(this) ? this : + null; + } + private Task InternalCreateCollectionAsync(bool ifNotExists, CancellationToken cancellationToken = default) { return this._client.CreateTableAsync(this.CollectionName, this._propertyReader.RecordDefinition.Properties, ifNotExists, cancellationToken); @@ -332,7 +358,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = PostgresConstants.DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -349,7 +375,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = PostgresConstants.DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs index 27fa7181bdc5..ef4e78cf9237 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs @@ -18,16 +18,21 @@ internal static class PostgresVectorStoreUtils /// The async enumerable to wrap. /// The name of the operation being performed. /// The name of the collection being operated on. + /// The name of the vector store system being operated on. /// An async enumerable that will throw a if an exception is thrown while iterating over the original enumerator. - public static async IAsyncEnumerable WrapAsyncEnumerableAsync(IAsyncEnumerable asyncEnumerable, string operationName, string? collectionName = null) + public static async IAsyncEnumerable WrapAsyncEnumerableAsync( + IAsyncEnumerable asyncEnumerable, + string operationName, + string? collectionName = null, + string? vectorStoreSystemName = null) { var enumerator = asyncEnumerable.ConfigureAwait(false).GetAsyncEnumerator(); - var nextResult = await GetNextAsync(enumerator, operationName, collectionName).ConfigureAwait(false); + var nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreSystemName).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.item; - nextResult = await GetNextAsync(enumerator, operationName, collectionName).ConfigureAwait(false); + nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreSystemName).ConfigureAwait(false); } } @@ -38,8 +43,13 @@ public static async IAsyncEnumerable WrapAsyncEnumerableAsync(IAsyncEnumer /// The enumerator to get the next result from. /// The name of the operation being performed. /// The name of the collection being operated on. + /// The name of the vector store system being operated on. /// A value indicating whether there are more results and the current string if true. - public static async Task<(T item, bool more)> GetNextAsync(ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, string operationName, string? collectionName = null) + public static async Task<(T item, bool more)> GetNextAsync( + ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, + string operationName, + string? collectionName = null, + string? vectorStoreSystemName = null) { try { @@ -50,7 +60,7 @@ public static async IAsyncEnumerable WrapAsyncEnumerableAsync(IAsyncEnumer { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = PostgresConstants.DatabaseName, + VectorStoreType = vectorStoreSystemName, CollectionName = collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs index bfac788a7cfd..051956ba964f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Qdrant vector store. /// @@ -18,8 +20,8 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; /// public class QdrantVectorStore : IVectorStore { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Qdrant"; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; /// Qdrant client that can be used to manage the collections and points in a Qdrant store. private readonly MockableQdrantClient _qdrantClient; @@ -48,6 +50,11 @@ internal QdrantVectorStore(MockableQdrantClient qdrantClient, QdrantVectorStoreO this._qdrantClient = qdrantClient; this._options = options ?? new QdrantVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreSystemName = "qdrant" + }; } /// @@ -88,7 +95,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._metadata.VectorStoreSystemName, OperationName = "ListCollections" }; } @@ -98,4 +105,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat yield return collection; } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(QdrantClient) ? this._qdrantClient.QdrantClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs index 760aeaae24f4..3217e6fa16cf 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs @@ -14,6 +14,8 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Qdrant as the underlying storage. /// @@ -25,6 +27,9 @@ public class QdrantVectorStoreRecordCollection : IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -38,9 +43,6 @@ public class QdrantVectorStoreRecordCollection : /// The default options for hybrid vector search. private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new(); - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Qdrant"; - /// The name of the upsert operation for telemetry purposes. private const string UpsertName = "Upsert"; @@ -128,6 +130,12 @@ internal QdrantVectorStoreRecordCollection(MockableQdrantClient qdrantClient, st this._propertyReader, this._options.HasNamedVectors); } + + this._collectionMetadata = new() + { + VectorStoreSystemName = "qdrant", + CollectionName = collectionName + }; } /// @@ -352,7 +360,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken c // Create point from record. var pointStruct = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, UpsertName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -371,7 +379,7 @@ async Task IVectorStoreRecordCollection.UpsertAsync(TRecord // Create point from record. var pointStruct = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, UpsertName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -390,7 +398,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -413,7 +421,7 @@ async IAsyncEnumerable IVectorStoreRecordCollection.UpsertB // Create points from records. var pointStructs = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, UpsertName, () => records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -471,7 +479,7 @@ private async IAsyncEnumerable GetBatchByPointIdAsync( } yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(pointStruct, new() { IncludeVectors = includeVectors })); @@ -532,7 +540,7 @@ public virtual async Task> VectorizedSearchAsync> HybridSearchAsync(TVect point, this._mapper, internalOptions.IncludeVectors, - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "Query")); return new VectorSearchResults(mappedResults.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(QdrantClient) ? this._qdrantClient.QdrantClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Run the given operation and wrap any with ."/> /// @@ -647,7 +668,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; @@ -671,7 +692,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs index a08fe1e86628..a682c84d615e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs @@ -14,6 +14,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Redis HashSets as the underlying storage. /// @@ -22,8 +24,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; public class RedisHashSetVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Redis"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -141,6 +143,13 @@ public RedisHashSetVectorStoreRecordCollection(IDatabase database, string collec // Default Mapper. this._mapper = new RedisHashSetVectorStoreRecordMapper(this._propertyReader); } + + this._collectionMetadata = new() + { + VectorStoreSystemName = "redis", + DatabaseName = database.Database.ToString(), + CollectionName = collectionName + }; } /// @@ -162,7 +171,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = "FT.INFO" }; @@ -251,7 +260,7 @@ await this.RunOperationAsync("FT.DROPINDEX", // Convert to the caller's data model. return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, operationName, () => @@ -309,7 +318,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken // Map. var redisHashSetRecord = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "HSET", () => this._mapper.MapFromDataToStorageModel(record)); @@ -377,7 +386,7 @@ public virtual async Task> VectorizedSearchAsync @@ -396,6 +405,19 @@ public virtual async Task> VectorizedSearchAsync(mappedResults.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(IDatabase) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Prefix the key with the collection name if the option is set. /// @@ -445,7 +467,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; @@ -468,7 +490,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs index af6a0a7d220f..6fa422fce726 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs @@ -17,6 +17,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Redis JSON as the underlying storage. /// @@ -25,8 +27,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; public class RedisJsonVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Redis"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -123,6 +125,13 @@ public RedisJsonVectorStoreRecordCollection(IDatabase database, string collectio // Default Mapper. this._mapper = new RedisJsonVectorStoreRecordMapper(this._propertyReader.KeyPropertyJsonName, this._jsonSerializerOptions); } + + this._collectionMetadata = new() + { + VectorStoreSystemName = "redis", + DatabaseName = database.Database.ToString(), + CollectionName = collectionName + }; } /// @@ -144,7 +153,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = "FT.INFO" }; @@ -233,7 +242,7 @@ await this.RunOperationAsync("FT.DROPINDEX", // Convert to the caller's data model. return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "GET", () => @@ -282,7 +291,7 @@ public virtual async IAsyncEnumerable GetBatchAsync(IEnumerable // Convert to the caller's data model. yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "MGET", () => @@ -326,7 +335,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken // Map. var redisJsonRecord = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "SET", () => @@ -360,7 +369,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable @@ -416,7 +425,7 @@ public virtual async Task> VectorizedSearchAsync @@ -438,6 +447,19 @@ public virtual async Task> VectorizedSearchAsync(mappedResults.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(IDatabase) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Prefix the key with the collection name if the option is set. /// @@ -486,7 +508,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; @@ -510,7 +532,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs index 4966917d3990..2366b889b647 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Redis vector store. /// @@ -18,8 +20,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; /// public class RedisVectorStore : IVectorStore { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Redis"; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; /// The redis database to read/write indices from. private readonly IDatabase _database; @@ -38,6 +40,12 @@ public RedisVectorStore(IDatabase database, RedisVectorStoreOptions? options = d this._database = database; this._options = options ?? new RedisVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreSystemName = "redis", + DatabaseName = database.Database.ToString() + }; } /// @@ -82,7 +90,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._metadata.VectorStoreSystemName, OperationName = OperationName }; } @@ -96,4 +104,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(IDatabase) ? this._metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs index d9481ffc467d..4c686b8305fb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; @@ -8,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.SqlServer; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// An implementation of backed by a SQL Server or Azure SQL database. /// @@ -16,6 +19,9 @@ public sealed class SqlServerVectorStore : IVectorStore private readonly string _connectionString; private readonly SqlServerVectorStoreOptions _options; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// /// Initializes a new instance of the class. /// @@ -31,6 +37,14 @@ public SqlServerVectorStore(string connectionString, SqlServerVectorStoreOptions this._options = options is not null ? new() { Schema = options.Schema } : SqlServerVectorStoreOptions.Defaults; + + var connectionStringBuilder = new SqlConnectionStringBuilder(connectionString); + + this._metadata = new() + { + VectorStoreSystemName = "microsoft.sql_server", + DatabaseName = connectionStringBuilder.InitialCatalog + }; } /// @@ -63,4 +77,16 @@ public async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancel yield return reader.GetString(reader.GetOrdinal("table_name")); } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs index 9b4ce3b29078..7e11cc02f37f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.SqlServer; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// An implementation of backed by a SQL Server or Azure SQL database. /// @@ -19,6 +21,9 @@ public sealed class SqlServerVectorStoreRecordCollection #pragma warning restore CA1711 : IVectorStoreRecordCollection where TKey : notnull { + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); private static readonly SqlServerVectorStoreRecordCollectionOptions s_defaultOptions = new(); @@ -88,6 +93,15 @@ public SqlServerVectorStoreRecordCollection( this._mapper = new RecordMapper(propertyReader); } + + var connectionStringBuilder = new SqlConnectionStringBuilder(connectionString); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "microsoft.sql_server", + DatabaseName = connectionStringBuilder.InitialCatalog, + CollectionName = name + }; } /// @@ -474,6 +488,18 @@ public async Task> VectorizedSearchAsync(T }, cancellationToken, "VectorizedSearch", this.CollectionName).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } + private async IAsyncEnumerable> ReadVectorSearchResultsAsync( SqlConnection connection, SqlCommand command, diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs index f5b9615884ff..a126a290fc8a 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Sqlite; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a SQLite vector store. /// @@ -18,6 +20,9 @@ namespace Microsoft.SemanticKernel.Connectors.Sqlite; /// public class SqliteVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// The connection string for the SQLite database represented by this . private readonly string _connectionString; @@ -35,6 +40,14 @@ public SqliteVectorStore(string connectionString, SqliteVectorStoreOptions? opti this._connectionString = connectionString; this._options = options ?? new(); + + var connectionStringBuilder = new SqliteConnectionStringBuilder(connectionString); + + this._metadata = new() + { + VectorStoreSystemName = "sqlite", + DatabaseName = connectionStringBuilder.DataSource + }; } /// @@ -100,4 +113,16 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat yield return reader.GetString(ordinal); } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs index 16dbd7238aca..53eec93755b1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs @@ -12,6 +12,8 @@ namespace Microsoft.SemanticKernel.Connectors.Sqlite; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses SQLite as the underlying storage. /// @@ -22,8 +24,8 @@ public class SqliteVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "SQLite"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// The connection string for the SQLite database represented by this . private readonly string _connectionString; @@ -112,6 +114,15 @@ public SqliteVectorStoreRecordCollection( this._vectorTableStoragePropertyNames = new(() => [this._propertyReader.KeyPropertyStoragePropertyName, .. this._propertyReader.VectorPropertyStoragePropertyNames]); this._mapper = this.InitializeMapper(); + + var connectionStringBuilder = new SqliteConnectionStringBuilder(connectionString); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "sqlite", + DatabaseName = connectionStringBuilder.DataSource, + CollectionName = collectionName + }; } /// @@ -338,6 +349,18 @@ public async Task DeleteBatchAsync(IEnumerable keys, CancellationToken c #endregion + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async ValueTask GetConnectionAsync(CancellationToken cancellationToken = default) @@ -555,7 +578,7 @@ private async Task InternalUpsertAsync(SqliteConnection connection, const string OperationName = "Upsert"; var storageModel = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -578,7 +601,7 @@ private IAsyncEnumerable InternalUpsertBatchAsync(SqliteConnection c const string OperationName = "UpsertBatch"; var storageModels = records.Select(record => VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -711,7 +734,7 @@ private TRecord GetAndMapRecord( } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors })); @@ -727,7 +750,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs index 1c45d1e3ac65..dfb03bde493f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Weaviate vector store. /// @@ -19,6 +21,9 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; /// public class WeaviateVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that is used to interact with Weaviate API. private readonly HttpClient _httpClient; @@ -40,6 +45,11 @@ public WeaviateVectorStore(HttpClient httpClient, WeaviateVectorStoreOptions? op this._httpClient = httpClient; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreSystemName = "weaviate" + }; } /// @@ -92,4 +102,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(HttpClient) ? this._httpClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 393b9a841cbb..6e488a3c56a7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -16,6 +16,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Weaviate as the underlying storage. /// @@ -24,8 +26,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; public class WeaviateVectorStoreRecordCollection : IVectorStoreRecordCollection, IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Weaviate"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -154,6 +156,12 @@ public WeaviateVectorStoreRecordCollection( // Assign mapper. this._mapper = this.InitializeMapper(); + + this._collectionMetadata = new() + { + VectorStoreSystemName = "weaviate", + CollectionName = collectionName + }; } /// @@ -270,7 +278,7 @@ public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken c } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(jsonObject!, new() { IncludeVectors = includeVectors })); @@ -312,7 +320,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable { var jsonObjects = records.Select(record => VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -395,6 +403,19 @@ public async Task> HybridSearchAsync(TVect return await this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.HybridScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(HttpClient) ? this._httpClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async Task> ExecuteQueryAsync(string query, bool includeVectors, string scorePropertyName, string operationName, CancellationToken cancellationToken) @@ -409,7 +430,7 @@ private async Task> ExecuteQueryAsync(string query, { throw new VectorStoreOperationException($"Error occurred during vector search. Response: {content}") { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -420,7 +441,7 @@ private async Task> ExecuteQueryAsync(string query, var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!, scorePropertyName); var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors })); @@ -483,7 +504,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs index 0533ab28c3f3..3493162d5095 100644 --- a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs @@ -22,6 +22,7 @@ public class PostgresVectorStoreRecordCollectionTests public PostgresVectorStoreRecordCollectionTests() { this._postgresClientMock = new Mock(MockBehavior.Strict); + this._postgresClientMock.Setup(l => l.DatabaseName).Returns("TestDatabase"); } [Fact] diff --git a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs index 33cfc005a7bc..8a89582fc0f8 100644 --- a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs @@ -26,6 +26,7 @@ public class PostgresVectorStoreTests public PostgresVectorStoreTests() { this._postgresClientMock = new Mock(MockBehavior.Strict); + this._postgresClientMock.Setup(l => l.DatabaseName).Returns("TestDatabase"); } [Fact] @@ -60,7 +61,10 @@ public void GetCollectionCallsFactoryIfProvided() var factoryMock = new Mock(MockBehavior.Strict); var collectionMock = new Mock>>(MockBehavior.Strict); var clientMock = new Mock(MockBehavior.Strict); + clientMock.Setup(x => x.DataSource).Returns(null); + clientMock.Setup(x => x.DatabaseName).Returns("TestDatabase"); + factoryMock .Setup(x => x.CreateVectorStoreRecordCollection>(It.IsAny(), TestCollectionName, null)) .Returns(collectionMock.Object); diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs index 117d3d1fcd4b..74c3dc49cf28 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs @@ -28,6 +28,7 @@ public class RedisHashSetVectorStoreRecordCollectionTests public RedisHashSetVectorStoreRecordCollectionTests() { this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); var batchMock = new Mock(); this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs index aa47dc512b8c..b729f1cba7a3 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs @@ -32,6 +32,7 @@ public class RedisJsonVectorStoreRecordCollectionTests public RedisJsonVectorStoreRecordCollectionTests() { this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); var batchMock = new Mock(); this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs index baf2564c81a2..9280051fd266 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs @@ -22,6 +22,7 @@ public class RedisVectorStoreTests public RedisVectorStoreTests() { this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); var batchMock = new Mock(); this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); diff --git a/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml b/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml index cd9bfbaa3ca7..6281be710692 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml +++ b/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml @@ -1,317 +1,86 @@  - - CP0001 - T:Microsoft.Extensions.VectorData.DeleteRecordOptions - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.UpsertRecordOptions - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.VectorSearchOptions - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.DeleteRecordOptions - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.UpsertRecordOptions - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.VectorSearchOptions - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.DeleteRecordOptions - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.UpsertRecordOptions - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.VectorSearchOptions - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IKeywordHybridSearch`1.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorStore.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IKeywordHybridSearch`1.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorStore.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IKeywordHybridSearch`1.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorStore.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj b/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj index f1dc235aa5bd..50aec26b5951 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj @@ -13,7 +13,7 @@ 9.0.0-preview.1.25161.1 9.0.0.0 - 9.0.0-preview.1.25078.1 + 9.0.0-preview.1.25161.1 Microsoft.Extensions.VectorData.Abstractions $(AssemblyName) Abstractions for vector database access. @@ -42,5 +42,6 @@ Microsoft.Extensions.VectorData.IVectorStoreRecordCollection<TKey, TRecord> + diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs index 53d2e062fcda..265366bf1377 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -26,4 +28,17 @@ Task> HybridSearchAsync( ICollection keywords, HybridSearchOptions? options = default, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs index 5368c5301828..f29d31ee49cd 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -22,4 +24,17 @@ Task> VectorizableTextSearchAsync( string searchText, VectorSearchOptions? options = default, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs index b2a5a54194a6..7a8149e6f69a 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -23,4 +25,17 @@ Task> VectorizedSearchAsync( TVector vector, VectorSearchOptions? options = default, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs index a3ac5466323c..d49af70a10b1 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; namespace Microsoft.Extensions.VectorData; @@ -38,4 +40,17 @@ IVectorStoreRecordCollection GetCollection(string /// The to monitor for cancellation requests. The default is . /// The list of names of all the collections in the vector store. IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs index f891dcba26b0..6d68fad0d13a 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -22,6 +23,7 @@ public interface IVectorStoreRecordCollection : IVectorizedSearch /// /// Gets the name of the collection. /// + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] string CollectionName { get; } /// diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs new file mode 100644 index 000000000000..eb7b89df5094 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class VectorStoreMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. + /// + public string? VectorStoreSystemName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } +} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs new file mode 100644 index 000000000000..f4b3a1aacfed --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class VectorStoreRecordCollectionMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. + /// + public string? VectorStoreSystemName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } + + /// + /// The name of a collection (table, container) within the database. + /// + public string? CollectionName { get; init; } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs new file mode 100644 index 000000000000..aa819b4b74bc --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs @@ -0,0 +1,433 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests; + +public class TelemetryHelpersTests +{ + [Fact] + public async Task RunOperationWithoutResultWorksWithActivityAndMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + bool operationExecuted = false; + async Task Operation() + { + await Task.Yield(); + operationExecuted = true; + } + + // Act + await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation); + + // Assert + Assert.True(operationExecuted); + + var activity = Assert.Single(activities); + + Assert.Equal($"{operationName} {collectionName}", activity.DisplayName); + Assert.Equal(operationName, activity.GetTagItem("db.operation.name")); + Assert.Equal(collectionName, activity.GetTagItem("db.collection.name")); + Assert.Equal(databaseName, activity.GetTagItem("db.namespace")); + Assert.Equal(systemName, activity.GetTagItem("db.system.name")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + Assert.Equal("db.client.operation.duration", metric.Name); + + this.AssertMetric(metric, operationName); + } + + [Fact] + public async Task RunOperationWithoutResultRecordsErrorOnExceptionAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + static Task Operation() => throw new InvalidOperationException("Test exception"); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => + TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation)); + + Assert.Equal("Test exception", exception.Message); + + var activity = Assert.Single(activities); + + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + this.AssertMetricError(metric); + } + + [Fact] + public async Task RunOperationWithResultWorksWithActivityAndMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + async static Task Operation() + { + await Task.Yield(); + return 42; + } + + // Act + var result = await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation); + + // Assert + Assert.Equal(42, result); + + var activity = Assert.Single(activities); + Assert.Equal($"{operationName} {collectionName}", activity.DisplayName); + Assert.Equal(operationName, activity.GetTagItem("db.operation.name")); + Assert.Equal(collectionName, activity.GetTagItem("db.collection.name")); + Assert.Equal(databaseName, activity.GetTagItem("db.namespace")); + Assert.Equal(systemName, activity.GetTagItem("db.system.name")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + Assert.Equal("db.client.operation.duration", metric.Name); + + this.AssertMetric(metric, operationName); + } + + [Fact] + public async Task RunOperationWithResultRecordsErrorOnExceptionAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + static Task Operation() => throw new InvalidOperationException("Test exception"); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => + TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation)); + + Assert.Equal("Test exception", exception.Message); + + var activity = Assert.Single(activities); + + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + this.AssertMetricError(metric); + } + + [Fact] + public async Task RunOperationWithAsyncEnumerableWorksWithActivityAndMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var result = new[] { 1, 2, 3 }; + + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + IAsyncEnumerable Operation() + { + return result.ToAsyncEnumerable(); + } + + // Act + var results = await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation, + default).ToListAsync(); + + // Assert + Assert.Equal(result, results); + + var activity = Assert.Single(activities); + + Assert.Equal($"{operationName} {collectionName}", activity.DisplayName); + Assert.Equal(operationName, activity.GetTagItem("db.operation.name")); + Assert.Equal(collectionName, activity.GetTagItem("db.collection.name")); + Assert.Equal(databaseName, activity.GetTagItem("db.namespace")); + Assert.Equal(systemName, activity.GetTagItem("db.system.name")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + Assert.Equal("db.client.operation.duration", metric.Name); + + this.AssertMetric(metric, operationName); + } + + [Fact] + public async Task RunOperationWithAsyncEnumerableRecordsErrorOnExceptionDuringEnumerationAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + async static IAsyncEnumerable Operation() + { + yield return 1; + await Task.Yield(); + throw new InvalidOperationException("Test exception"); + } + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation, + default).ToListAsync()); + + Assert.Equal("Test exception", exception.Message); + + var activity = Assert.Single(activities); + + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + this.AssertMetricError(metric); + } + + #region private + + private void AssertMetric(Metric? metric, string operationName) + { + var metricExists = false; + + if (metric is not null) + { + foreach (var point in metric.GetMetricPoints()) + { + var histogramExists = point.GetHistogramSum() > 0; + var tagExists = false; + + foreach (var tag in point.Tags) + { + if (tag.Key == "db.operation.name" && tag.Value?.ToString() == operationName) + { + tagExists = true; + } + } + + metricExists = histogramExists && tagExists; + } + } + + Assert.True(metricExists); + } + + private void AssertMetricError(Metric? metric) + { + var metricErrorExists = false; + + if (metric is not null) + { + foreach (var point in metric.GetMetricPoints()) + { + foreach (var tag in point.Tags) + { + metricErrorExists = + tag.Key == "error.type" && + tag.Value?.ToString() == typeof(InvalidOperationException).FullName; + } + } + } + + Assert.True(metricErrorExists); + } + + #endregion +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj b/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj index d374de2022ba..4cde8a374c10 100644 --- a/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj @@ -12,6 +12,7 @@ + @@ -29,7 +30,6 @@ - diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs new file mode 100644 index 000000000000..b538ac2b9534 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorSearch; + +public class OpenTelemetryKeywordHybridSearchTests +{ + [Fact] + public async Task HybridSearchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List> { new("record1", 0.95f) }.ToAsyncEnumerable()); + + var mockInnerSearch = new Mock>(); + + mockInnerSearch + .Setup(s => s.HybridSearchAsync(It.IsAny(), It.IsAny>(), null, default)) + .ReturnsAsync(expectedResults); + + mockInnerSearch + .Setup(s => s.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + + var innerSearch = mockInnerSearch.Object; + var keywordHybridSearch = innerSearch + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var vector = new float[] { 1.0f }; + + // Act + var result = await keywordHybridSearch.HybridSearchAsync(vector, ["keyword1"]); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("hybrid_search testcollection", activity.DisplayName); + Assert.Equal("hybrid_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs new file mode 100644 index 000000000000..7fdbb3eb6da0 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorSearch; + +public class OpenTelemetryVectorizableTextSearchTests +{ + [Fact] + public async Task VectorizableTextSearchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List> { new("record1", 0.90f) }.ToAsyncEnumerable()); + + var mockInnerSearch = new Mock>(); + + mockInnerSearch + .Setup(s => s.VectorizableTextSearchAsync("test query", null, default)) + .ReturnsAsync(expectedResults); + + mockInnerSearch + .Setup(s => s.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + + var innerSearch = mockInnerSearch.Object; + var vectorizableTextSearch = innerSearch + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorizableTextSearch.VectorizableTextSearchAsync("test query"); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("vectorizable_text_search testcollection", activity.DisplayName); + Assert.Equal("vectorizable_text_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs new file mode 100644 index 000000000000..28c74596a48e --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorSearch; + +public class OpenTelemetryVectorizedSearchTests +{ + [Fact] + public async Task VectorizedSearchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List> { new("record1", 0.95f) }.ToAsyncEnumerable()); + + var mockInnerSearch = new Mock>(); + + mockInnerSearch + .Setup(s => s.VectorizedSearchAsync(It.IsAny(), null, default)) + .ReturnsAsync(expectedResults); + + mockInnerSearch + .Setup(s => s.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + + var innerSearch = mockInnerSearch.Object; + var vectorizedSearch = innerSearch + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var vector = new float[] { 1.0f, 2.0f }; + + // Act + var result = await vectorizedSearch.VectorizedSearchAsync(vector); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("vectorized_search testcollection", activity.DisplayName); + Assert.Equal("vectorized_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs index 294ee64555a0..7bc32d9a3dfc 100644 --- a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs @@ -32,23 +32,6 @@ public void ConstructorThrowsOnNullLogger() Assert.Throws(() => new LoggingVectorStoreRecordCollection(innerCollection, null!)); } - [Fact] - public void CollectionNameReturnsInnerCollectionName() - { - // Arrange - var innerCollection = new Mock>(); - innerCollection.Setup(c => c.CollectionName).Returns("test"); - var logger = new Mock().Object; - var decorator = new LoggingVectorStoreRecordCollection(innerCollection.Object, logger); - - // Act - var name = decorator.CollectionName; - - // Assert - Assert.Equal("test", name); - innerCollection.Verify(c => c.CollectionName, Times.Once()); - } - [Fact] public async Task CollectionExistsDelegatesToInnerCollectionAsync() { diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs new file mode 100644 index 000000000000..90c6f2569294 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs @@ -0,0 +1,418 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorStorage; + +public class OpenTelemetryVectorStoreRecordCollectionTests +{ + private readonly Mock> _mockCollection; + + public OpenTelemetryVectorStoreRecordCollectionTests() + { + this._mockCollection = new(); + + this._mockCollection.Setup(c => c.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + } + + [Fact] + public async Task CollectionExistsWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.CollectionExistsAsync(default)).ReturnsAsync(true); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.CollectionExistsAsync(); + + // Assert + Assert.True(result); + + var activity = Assert.Single(activities); + + Assert.Equal("collection_exists testcollection", activity.DisplayName); + Assert.Equal("collection_exists", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task CreateCollectionWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.CreateCollectionAsync(default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.CreateCollectionAsync(); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("create_collection testcollection", activity.DisplayName); + Assert.Equal("create_collection", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task CreateCollectionIfNotExistsWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.CreateCollectionIfNotExistsAsync(default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.CreateCollectionIfNotExistsAsync(); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("create_collection_if_not_exists testcollection", activity.DisplayName); + Assert.Equal("create_collection_if_not_exists", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task DeleteCollectionWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.DeleteCollectionAsync(default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.DeleteCollectionAsync(); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("delete_collection testcollection", activity.DisplayName); + Assert.Equal("delete_collection", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task GetWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedRecord = new object(); + + this._mockCollection.Setup(c => c.GetAsync("key1", null, default)).ReturnsAsync(expectedRecord); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.GetAsync("key1"); + + // Assert + Assert.Equal(expectedRecord, result); + + var activity = Assert.Single(activities); + + Assert.Equal("get testcollection", activity.DisplayName); + Assert.Equal("get", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task GetBatchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + object[] expectedRecords = [new(), new()]; + string[] keys = ["key1", "key2"]; + + this._mockCollection.Setup(c => c.GetBatchAsync(keys, null, default)) + .Returns(expectedRecords.ToAsyncEnumerable()); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.GetBatchAsync(keys).ToListAsync(); + + // Assert + Assert.Equal(expectedRecords, result); + + var activity = Assert.Single(activities); + + Assert.Equal("get_batch testcollection", activity.DisplayName); + Assert.Equal("get_batch", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task DeleteWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.DeleteAsync("key1", default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.DeleteAsync("key1"); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("delete testcollection", activity.DisplayName); + Assert.Equal("delete", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task DeleteBatchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + string[] keys = ["key1", "key2"]; + + this._mockCollection.Setup(c => c.DeleteBatchAsync(keys, default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.DeleteBatchAsync(keys); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("delete_batch testcollection", activity.DisplayName); + Assert.Equal("delete_batch", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task UpsertWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedKey = "key1"; + + this._mockCollection.Setup(c => c.UpsertAsync(It.IsAny(), default)).ReturnsAsync(expectedKey); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.UpsertAsync(new object()); + + // Assert + Assert.Equal(expectedKey, result); + + var activity = Assert.Single(activities); + + Assert.Equal("upsert testcollection", activity.DisplayName); + Assert.Equal("upsert", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task UpsertBatchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + string[] expectedKeys = ["key1", "key2"]; + + this._mockCollection.Setup(c => c.UpsertBatchAsync(It.IsAny>(), default)) + .Returns(expectedKeys.ToAsyncEnumerable()); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.UpsertBatchAsync([new object(), new object()]).ToListAsync(); + + // Assert + Assert.Equal(expectedKeys, result); + + var activity = Assert.Single(activities); + + Assert.Equal("upsert_batch testcollection", activity.DisplayName); + Assert.Equal("upsert_batch", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task VectorizedSearchAsyncWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List>().ToAsyncEnumerable()); + + this._mockCollection.Setup(c => c.VectorizedSearchAsync(It.IsAny(), null, default)) + .ReturnsAsync(expectedResults); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var vector = new float[] { 1.0f }; + + // Act + var result = await vectorStoreCollection.VectorizedSearchAsync(vector); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("vectorized_search testcollection", activity.DisplayName); + Assert.Equal("vectorized_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs new file mode 100644 index 000000000000..4ab0a1bb8b5c --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorStorage; + +public class OpenTelemetryVectorStoreTests +{ + [Fact] + public async Task ListCollectionNamesWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + string[] colletions = ["col1", "col2"]; + var mockInnerStore = new Mock(); + + mockInnerStore.Setup(s => s.ListCollectionNamesAsync(default)) + .Returns(colletions.ToAsyncEnumerable()); + + mockInnerStore.Setup(s => s.GetService(typeof(VectorStoreMetadata), It.IsAny())) + .Returns(new VectorStoreMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb" + }); + + var vectorStore = mockInnerStore.Object + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStore.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Equal(colletions, result); + + var activity = Assert.Single(activities); + + Assert.Equal("list_collection_names testdb", activity.DisplayName); + Assert.Equal("list_collection_names", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs b/dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs new file mode 100644 index 000000000000..f053bba48d44 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.VectorData; + +/// +/// OpenTelemetry constants from Semantic Conventions for database calls and systems: +/// . +/// . +/// . +/// +internal static class OpenTelemetryConstants +{ + public const string DefaultSourceName = "Experimental.Microsoft.Extensions.VectorData"; + + public const string DbSystemName = "db.system.name"; + public const string DbCollectionName = "db.collection.name"; + public const string DbNamespace = "db.namespace"; + public const string DbOperationName = "db.operation.name"; + + public const string DbOperationDurationMetricName = "db.client.operation.duration"; + public const string DbOperationDurationMetricDescription = "Duration of database client operations."; + + public const string ErrorType = "error.type"; + + public const string SecondsUnit = "s"; + + public static string GetSourceNameOrDefault(string? sourceName) => !string.IsNullOrWhiteSpace(sourceName) ? sourceName! : DefaultSourceName; + + public static string GetActivityName(string operationName, string? collectionName, string? namespaceName) + { + if (!string.IsNullOrWhiteSpace(collectionName)) + { + return $"{operationName} {collectionName}"; + } + + if (!string.IsNullOrWhiteSpace(namespaceName)) + { + return $"{operationName} {namespaceName}"; + } + + return operationName; + } +} diff --git a/dotnet/src/Connectors/VectorData/TelemetryHelpers.cs b/dotnet/src/Connectors/VectorData/TelemetryHelpers.cs new file mode 100644 index 000000000000..2824f67c608c --- /dev/null +++ b/dotnet/src/Connectors/VectorData/TelemetryHelpers.cs @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// +/// Class with helper methods to run operations with telemetry. +/// +internal static class TelemetryHelpers +{ + /// + /// Method to run operation which doesn't return a result. + /// + internal static async Task RunOperationAsync( + ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Func operation) + { + using var activity = activitySource.StartActivity(operationName, collectionName, databaseName, vectorStoreSystemName); + Stopwatch? stopwatch = operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + Exception? exception = null; + + try + { + await operation().ConfigureAwait(false); + } + catch (Exception ex) + { + exception = ex; + throw; + } + finally + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + exception, + stopwatch); + } + } + + /// + /// Method to run operation which returns the result. + /// + internal static async Task RunOperationAsync( + ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Func> operation) + { + using var activity = activitySource.StartActivity(operationName, collectionName, databaseName, vectorStoreSystemName); + Stopwatch? stopwatch = operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + Exception? exception = null; + + try + { + return await operation().ConfigureAwait(false); + } + catch (Exception ex) + { + exception = ex; + throw; + } + finally + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + exception, + stopwatch); + } + } + + /// + /// Method to run operation which returns async enumeration. + /// + internal static async IAsyncEnumerable RunOperationAsync( + ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Func> operation, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var activity = activitySource.StartActivity(operationName, collectionName, databaseName, vectorStoreSystemName); + Stopwatch? stopwatch = operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + IAsyncEnumerator enumerator; + + try + { + enumerator = operation().GetAsyncEnumerator(cancellationToken); + } + catch (Exception ex) + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + ex, + stopwatch); + + throw; + } + + Exception? exception = null; + + try + { + while (true) + { + try + { + if (!await enumerator.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + } + catch (Exception ex) + { + exception = ex; + throw; + } + + yield return enumerator.Current; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + finally + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + exception, + stopwatch); + + await enumerator.DisposeAsync().ConfigureAwait(false); + } + } + + #region private + + private static Activity? StartActivity( + this ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName) + { + Activity? activity = null; + + if (activitySource.HasListeners()) + { + activity = activitySource.StartActivity( + OpenTelemetryConstants.GetActivityName(operationName, collectionName, databaseName), + ActivityKind.Client); + + var tags = GetTags(operationName, collectionName, databaseName, vectorStoreSystemName); + + foreach (var tag in tags) + { + activity?.SetTag(tag.Key, tag.Value); + } + } + + return activity; + } + + private static void TraceOperationCompletion( + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Activity? activity, + Exception? exception, + Stopwatch? stopwatch) + { + if (operationDurationHistogram.Enabled && stopwatch is not null) + { + var tags = TelemetryHelpers.GetTags(operationName, collectionName, databaseName, vectorStoreSystemName); + + if (exception is not null) + { + tags.Add(OpenTelemetryConstants.ErrorType, exception.GetType().FullName); + } + + operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + + if (exception is not null) + { + activity? + .SetTag(OpenTelemetryConstants.ErrorType, exception.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, exception.Message); + } + } + + private static TagList GetTags( + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName) + { + TagList tags = default; + + tags.Add(OpenTelemetryConstants.DbOperationName, operationName); + + if (!string.IsNullOrWhiteSpace(collectionName)) + { + tags.Add(OpenTelemetryConstants.DbCollectionName, collectionName); + } + + if (!string.IsNullOrWhiteSpace(databaseName)) + { + tags.Add(OpenTelemetryConstants.DbNamespace, databaseName); + } + + if (!string.IsNullOrWhiteSpace(vectorStoreSystemName)) + { + tags.Add(OpenTelemetryConstants.DbSystemName, vectorStoreSystemName); + } + + return tags; + } + + #endregion +} diff --git a/dotnet/src/Connectors/VectorData/VectorData.csproj b/dotnet/src/Connectors/VectorData/VectorData.csproj index 31e84c2533ce..f80443a6c310 100644 --- a/dotnet/src/Connectors/VectorData/VectorData.csproj +++ b/dotnet/src/Connectors/VectorData/VectorData.csproj @@ -59,4 +59,8 @@ + + + + diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs index c05ffde310cc..d5c3d28acb39 100644 --- a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs +++ b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -44,4 +45,14 @@ public Task> HybridSearchAsync(TVector vec nameof(HybridSearchAsync), () => this._innerSearch.HybridSearchAsync(vector, keywords, options, cancellationToken)); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs index 0dc81080e496..a60b6dd12388 100644 --- a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs +++ b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -43,4 +44,14 @@ public Task> VectorizableTextSearchAsync(string sea nameof(VectorizableTextSearchAsync), () => this._innerSearch.VectorizableTextSearchAsync(searchText, options, cancellationToken)); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs index f0198534d421..3e6c099b2282 100644 --- a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs +++ b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -43,4 +44,14 @@ public Task> VectorizedSearchAsync(TVector nameof(VectorizedSearchAsync), () => this._innerSearch.VectorizedSearchAsync(vector, options, cancellationToken)); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs new file mode 100644 index 000000000000..3c4aa653dcc2 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating search that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryKeywordHybridSearch : IKeywordHybridSearch, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IKeywordHybridSearch _innerSearch; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryKeywordHybridSearch(IKeywordHybridSearch innerSearch, string? sourceName = null) + { + this._innerSearch = innerSearch ?? throw new ArgumentNullException(nameof(innerSearch)); + + if (this._innerSearch.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new(telemetrySourceName); + this._meter = new(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public Task> HybridSearchAsync(TVector vector, ICollection keywords, HybridSearchOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "hybrid_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerSearch.HybridSearchAsync(vector, keywords, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = serviceType ?? throw new ArgumentNullException(nameof(serviceType)); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs new file mode 100644 index 000000000000..87445776f8e9 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryKeywordHybridSearchBuilderExtensions +{ + /// Adds OpenTelemetry support to the keyword hybrid search pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static KeywordHybridSearchBuilder UseOpenTelemetry( + this KeywordHybridSearchBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerSearch, services) => + { + return new OpenTelemetryKeywordHybridSearch(innerSearch, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs new file mode 100644 index 000000000000..f9b2cc8397dc --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vectorizable text search that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryVectorizableTextSearch : IVectorizableTextSearch, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorizableTextSearch _innerSearch; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorizableTextSearch(IVectorizableTextSearch innerSearch, string? sourceName = null) + { + this._innerSearch = innerSearch ?? throw new ArgumentNullException(nameof(innerSearch)); + + if (this._innerSearch.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(telemetrySourceName); + this._meter = new Meter(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public Task> VectorizableTextSearchAsync( + string searchText, + VectorSearchOptions? options = default, + CancellationToken cancellationToken = default) + { + const string OperationName = "vectorizable_text_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerSearch.VectorizableTextSearchAsync(searchText, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = serviceType ?? throw new ArgumentNullException(nameof(serviceType)); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs new file mode 100644 index 000000000000..4f89fc2fcc97 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorizableTextSearchBuilderExtensions +{ + /// Adds OpenTelemetry support to the vectorizable text search pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorizableTextSearchBuilder UseOpenTelemetry( + this VectorizableTextSearchBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerSearch, services) => + { + return new OpenTelemetryVectorizableTextSearch(innerSearch, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs new file mode 100644 index 000000000000..7896b5664546 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vectorized search that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryVectorizedSearch : IVectorizedSearch, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorizedSearch _innerSearch; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorizedSearch(IVectorizedSearch innerSearch, string? sourceName = null) + { + this._innerSearch = innerSearch ?? throw new ArgumentNullException(nameof(innerSearch)); + + if (this._innerSearch.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(telemetrySourceName); + this._meter = new Meter(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public Task> VectorizedSearchAsync( + TVector vector, + VectorSearchOptions? options = default, + CancellationToken cancellationToken = default) + { + const string OperationName = "vectorized_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerSearch.VectorizedSearchAsync(vector, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = serviceType ?? throw new ArgumentNullException(nameof(serviceType)); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs new file mode 100644 index 000000000000..56390ff4305c --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorizedSearchBuilderExtensions +{ + /// Adds OpenTelemetry support to the vectorized search pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorizedSearchBuilder UseOpenTelemetry( + this VectorizedSearchBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerSearch, services) => + { + return new OpenTelemetryVectorizedSearch(innerSearch, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs index ea2bca4ca106..7fd38761f352 100644 --- a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs +++ b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -50,4 +51,14 @@ public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cance () => this._innerStore.ListCollectionNamesAsync(cancellationToken), cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerStore.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs index 3d6919280861..36e31d1cfd0d 100644 --- a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -39,6 +40,7 @@ public LoggingVectorStoreRecordCollection(IVectorStoreRecordCollection + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] public string CollectionName => this._innerCollection.CollectionName; /// @@ -114,6 +116,16 @@ public IAsyncEnumerable GetBatchAsync(IEnumerable keys, GetRecord cancellationToken); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerCollection.GetService(serviceType, serviceKey); + } + /// public Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs new file mode 100644 index 000000000000..edf647203da4 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vector store that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryVectorStore : IVectorStore, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorStore _innerStore; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string _telemetrySourceName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorStore(IVectorStore innerStore, string? sourceName = null) + { + this._innerStore = innerStore ?? throw new ArgumentNullException(nameof(innerStore)); + + if (this._innerStore.GetService(typeof(VectorStoreMetadata)) is VectorStoreMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + } + + this._telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(this._telemetrySourceName); + this._meter = new Meter(this._telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public IVectorStoreRecordCollection GetCollection(string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition = null) where TKey : notnull + => new OpenTelemetryVectorStoreRecordCollection( + this._innerStore.GetCollection(name, vectorStoreRecordDefinition), + this._telemetrySourceName); + + /// + public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "list_collection_names"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + null, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerStore.ListCollectionNamesAsync(cancellationToken), + cancellationToken); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerStore.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs new file mode 100644 index 000000000000..5f47d0708349 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorStoreBuilderExtensions +{ + /// Adds OpenTelemetry support to the vector store pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorStoreBuilder UseOpenTelemetry( + this VectorStoreBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerStore, services) => + { + return new OpenTelemetryVectorStore(innerStore, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs new file mode 100644 index 000000000000..830939fec0e4 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs @@ -0,0 +1,257 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vector store record collection that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class OpenTelemetryVectorStoreRecordCollection : IVectorStoreRecordCollection, IDisposable where TKey : notnull +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorStoreRecordCollection _innerCollection; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorStoreRecordCollection(IVectorStoreRecordCollection innerCollection, string? sourceName = null) + { + this._innerCollection = innerCollection ?? throw new ArgumentNullException(nameof(innerCollection)); + + if (this._innerCollection.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(telemetrySourceName); + this._meter = new Meter(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] + public string CollectionName => this._innerCollection.CollectionName; + + /// + public Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "collection_exists"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.CollectionExistsAsync(cancellationToken)); + } + + /// + public Task CreateCollectionAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "create_collection"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.CreateCollectionAsync(cancellationToken)); + } + + /// + public Task CreateCollectionIfNotExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "create_collection_if_not_exists"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.CreateCollectionIfNotExistsAsync(cancellationToken)); + } + + /// + public Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + const string OperationName = "delete"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.DeleteAsync(key, cancellationToken)); + } + + /// + public Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + const string OperationName = "delete_batch"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.DeleteBatchAsync(keys, cancellationToken)); + } + + /// + public Task DeleteCollectionAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "delete_collection"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.DeleteCollectionAsync(cancellationToken)); + } + + /// + public Task GetAsync(TKey key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "get"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.GetAsync(key, options, cancellationToken)); + } + + /// + public IAsyncEnumerable GetBatchAsync(IEnumerable keys, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "get_batch"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.GetBatchAsync(keys, options, cancellationToken), + cancellationToken); + } + + /// + public Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + const string OperationName = "upsert"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.UpsertAsync(record, cancellationToken)); + } + + /// + public IAsyncEnumerable UpsertBatchAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + const string OperationName = "upsert_batch"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.UpsertBatchAsync(records, cancellationToken), + cancellationToken); + } + + /// + public Task> VectorizedSearchAsync(TVector vector, VectorSearchOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "vectorized_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.VectorizedSearchAsync(vector, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerCollection.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs new file mode 100644 index 000000000000..06bb83404570 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorStoreRecordCollectionBuilderExtensions +{ + /// Adds OpenTelemetry support to the vector store record collection pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorStoreRecordCollectionBuilder UseOpenTelemetry( + this VectorStoreRecordCollectionBuilder builder, + string? sourceName = null) + where TKey : notnull + { + Verify.NotNull(builder); + + return builder.Use((innerCollection, services) => + { + return new OpenTelemetryVectorStoreRecordCollection(innerCollection, sourceName); + }); + } +} diff --git a/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs b/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs index 143c61f69e5f..ea852dbb36d7 100644 --- a/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs @@ -108,6 +108,16 @@ public async Task> VectorizableTextSearchAsync(stri return await vectorizedSearch.VectorizedSearchAsync(vectorizedQuery, options, cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + vectorizedSearch.GetService(serviceType, serviceKey); + } } /// diff --git a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs index b39976adbebf..17609c979d78 100644 --- a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs +++ b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs @@ -18,6 +18,16 @@ public Task> VectorizableTextSearchAsync(string sea return Task.FromResult(new VectorSearchResults(this._searchResults)); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + null; + } + private static async IAsyncEnumerable> ToAsyncEnumerable(IEnumerable> searchResults) { foreach (var result in searchResults) diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs index c01fe06eddf4..9c56a71974ca 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs @@ -131,6 +131,16 @@ public async Task> VectorizableTextSearchAsync(stri var vectorizedQuery = await textEmbeddingGeneration!.GenerateEmbeddingAsync(searchText, cancellationToken: cancellationToken).ConfigureAwait(false); return await vectorizedSearch.VectorizedSearchAsync(vectorizedQuery, options, cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + vectorizedSearch.GetService(serviceType, serviceKey); + } } /// diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs index c2d71d49281b..68c5d0100410 100644 --- a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs @@ -66,7 +66,11 @@ public async Task UpsertBatchIsAtomic() VectorStoreOperationException ex = await Assert.ThrowsAsync(() => collection.UpsertBatchAsync(inserted).ToArrayAsync().AsTask()); Assert.Equal("UpsertBatch", ex.OperationName); - Assert.Equal(collection.CollectionName, ex.CollectionName); + + var metadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; + + Assert.NotNull(metadata?.CollectionName); + Assert.Equal(metadata.CollectionName, ex.CollectionName); // Make sure that no records were inserted! Assert.Empty(await collection.GetBatchAsync(keys).ToArrayAsync()); diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 16f8679df842..34be5be73c97 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -86,7 +86,17 @@ private async Task CreateCollection() try { Assert.True(await collection.CollectionExistsAsync()); - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; +#pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + Assert.NotNull(collectionMetadata); + Assert.NotNull(collectionMetadata.VectorStoreSystemName); + Assert.NotNull(collectionMetadata.DatabaseName); + Assert.NotNull(collectionMetadata.CollectionName); + + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); } finally { @@ -103,7 +113,17 @@ private async Task CreateCollectionIfNotExistsMoreThanOnce() try { Assert.True(await collection.CollectionExistsAsync()); - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; +#pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + Assert.NotNull(collectionMetadata); + Assert.NotNull(collectionMetadata.VectorStoreSystemName); + Assert.NotNull(collectionMetadata.DatabaseName); + Assert.NotNull(collectionMetadata.CollectionName); + + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); await collection.CreateCollectionIfNotExistsAsync(); } @@ -122,7 +142,17 @@ private async Task CreateCollectionMoreThanOnce() try { Assert.True(await collection.CollectionExistsAsync()); - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; +#pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + Assert.NotNull(collectionMetadata); + Assert.NotNull(collectionMetadata.VectorStoreSystemName); + Assert.NotNull(collectionMetadata.DatabaseName); + Assert.NotNull(collectionMetadata.CollectionName); + + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); await collection.CreateCollectionIfNotExistsAsync();