From 762adb788ba5607e1ced38aa9f38da69fc01927f Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 14:44:25 -0700 Subject: [PATCH 01/19] Added metadata and GetService for IVectorStore --- .../TextEmbeddingVectorStore.cs | 10 +++++ .../AzureAISearchVectorStore.cs | 38 +++++++++++++++---- .../AzureCosmosDBMongoDBVectorStore.cs | 24 ++++++++++++ .../AzureCosmosDBNoSQLVectorStore.cs | 24 ++++++++++++ .../InMemoryVectorStore.cs | 28 ++++++++++++++ .../MongoDBVectorStore.cs | 24 ++++++++++++ .../PineconeVectorStore.cs | 26 ++++++++++++- .../PostgresVectorStore.cs | 34 +++++++++++++++++ .../QdrantVectorStore.cs | 26 +++++++++++-- .../RedisVectorStore.cs | 27 +++++++++++-- .../SqlServerVectorStore.cs | 26 +++++++++++++ .../SqliteVectorStore.cs | 24 ++++++++++++ .../WeaviateVectorStore.cs | 23 +++++++++++ .../VectorData.Abstractions.csproj | 1 + .../KeywordHybridSearchMetadata.cs | 28 ++++++++++++++ .../VectorizableTextSearchMetadata.cs | 28 ++++++++++++++ .../VectorSearch/VectorizedSearchMetadata.cs | 28 ++++++++++++++ .../VectorStorage/IVectorStore.cs | 13 +++++++ .../VectorStorage/VectorStoreMetadata.cs | 23 +++++++++++ .../VectorStoreRecordCollectionMetadata.cs | 28 ++++++++++++++ 20 files changed, 468 insertions(+), 15 deletions(-) create mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs create mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs create mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs create mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs create mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs diff --git a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs index 6848b38af48f..dba6e77f398b 100644 --- a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs +++ b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs @@ -40,6 +40,16 @@ public IVectorStoreRecordCollection GetCollection( return embeddingStore; } + /// + public virtual object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._decoratedVectorStore.GetService(serviceType, serviceKey); + } + /// public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) { diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs index 5329cdf3cee4..4d551c3cb1e7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Azure AI Search vector store. /// @@ -19,8 +21,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; /// public class AzureAISearchVectorStore : IVectorStore { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureAISearch"; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; /// Azure AI Search client that can be used to manage the list of indices in an Azure AI Search Service. private readonly SearchIndexClient _searchIndexClient; @@ -39,6 +41,12 @@ public AzureAISearchVectorStore(SearchIndexClient searchIndexClient, AzureAISear this._searchIndexClient = searchIndexClient; this._options = options ?? new AzureAISearchVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreName = "azure.aisearch", + DatabaseName = searchIndexClient.ServiceName + }; } /// @@ -75,22 +83,38 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat var indexNamesEnumerable = this._searchIndexClient.GetIndexNamesAsync(cancellationToken).ConfigureAwait(false); var indexNamesEnumerator = indexNamesEnumerable.GetAsyncEnumerator(); - var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator).ConfigureAwait(false); + var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.name; - nextResult = await GetNextIndexNameAsync(indexNamesEnumerator).ConfigureAwait(false); + nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata).ConfigureAwait(false); } } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(SearchIndexClient) ? this._searchIndexClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Helper method to get the next index name from the enumerator with a try catch around the move next call to convert /// any to , since try catch is not supported /// around a yield return. /// /// The enumerator to get the next result from. + /// Metadata about vector store. /// A value indicating whether there are more results and the current string if true. - private static async Task<(string name, bool more)> GetNextIndexNameAsync(ConfiguredCancelableAsyncEnumerable.Enumerator enumerator) + private static async Task<(string name, bool more)> GetNextIndexNameAsync( + ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, + VectorStoreMetadata metadata) { const string OperationName = "GetIndexNames"; @@ -103,7 +127,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = metadata.VectorStoreName, OperationName = OperationName }; } @@ -111,7 +135,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = metadata.VectorStoreName, OperationName = OperationName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs index 76dc9e8500a4..4c7efaba0aa4 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs @@ -9,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Azure CosmosDB MongoDB vector store. /// @@ -17,6 +19,9 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; /// public class AzureCosmosDBMongoDBVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that can be used to manage the collections in Azure CosmosDB MongoDB. private readonly IMongoDatabase _mongoDatabase; @@ -34,6 +39,12 @@ public AzureCosmosDBMongoDBVectorStore(IMongoDatabase mongoDatabase, AzureCosmos this._mongoDatabase = mongoDatabase; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreName = "azure.cosmosdbmongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName + }; } /// @@ -75,4 +86,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs index 39320e0a8ae2..011b69ff95f8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs @@ -9,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Azure CosmosDB NoSQL vector store. /// @@ -17,6 +19,9 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; /// public class AzureCosmosDBNoSQLVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that can be used to manage the collections in Azure CosmosDB NoSQL. private readonly Database _database; @@ -34,6 +39,12 @@ public AzureCosmosDBNoSQLVectorStore(Database database, AzureCosmosDBNoSQLVector this._database = database; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreName = "azure.cosmosdbnosql", + DatabaseName = database.Id + }; } /// @@ -84,4 +95,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(Database) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs index 2db7013b0d27..86b676d363df 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs @@ -9,11 +9,16 @@ namespace Microsoft.SemanticKernel.Connectors.InMemory; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Service for storing and retrieving vector records, and managing vector record collections, that uses an in memory dictionary as the underlying storage. /// public sealed class InMemoryVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// Internal storage for the record collection. private readonly ConcurrentDictionary> _internalCollection; @@ -26,6 +31,11 @@ public sealed class InMemoryVectorStore : IVectorStore public InMemoryVectorStore() { this._internalCollection = new(); + + this._metadata = new() + { + VectorStoreName = "inmemory" + }; } /// @@ -35,6 +45,11 @@ public InMemoryVectorStore() internal InMemoryVectorStore(ConcurrentDictionary> internalCollection) { this._internalCollection = internalCollection; + + this._metadata = new() + { + VectorStoreName = "inmemory" + }; } /// @@ -59,4 +74,17 @@ public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cance { return this._internalCollection.Keys.ToAsyncEnumerable(); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(ConcurrentDictionary>) ? this._internalCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs index 27169e3e9557..35ba7f0a2ba8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStore.cs @@ -9,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a MongoDB vector store. /// @@ -17,6 +19,9 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; /// public class MongoDBVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that can be used to manage the collections in MongoDB. private readonly IMongoDatabase _mongoDatabase; @@ -34,6 +39,12 @@ public MongoDBVectorStore(IMongoDatabase mongoDatabase, MongoDBVectorStoreOption this._mongoDatabase = mongoDatabase; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreName = "mongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName + }; } /// @@ -75,4 +86,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs index 4f79810e641b..4668531ea5cb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Pinecone vector store. /// @@ -19,12 +21,14 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; /// public class PineconeVectorStore : IVectorStore { - private const string DatabaseName = "Pinecone"; private const string ListCollectionsName = "ListCollections"; private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreOptions _options; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// /// Initializes a new instance of the class. /// @@ -36,6 +40,11 @@ public PineconeVectorStore(Sdk.PineconeClient pineconeClient, PineconeVectorStor this._pineconeClient = pineconeClient; this._options = options ?? new PineconeVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreName = "pinecone" + }; } /// @@ -73,7 +82,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._metadata.VectorStoreName, OperationName = ListCollectionsName }; } @@ -83,4 +92,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat yield return collection.Name; } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(Sdk.PineconeClient) ? this._pineconeClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs index 0f61e692ae7f..ba52e03e22b3 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs @@ -8,6 +8,8 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Represents a vector store implementation using PostgreSQL. /// @@ -17,6 +19,9 @@ public class PostgresVectorStore : IVectorStore private readonly NpgsqlDataSource? _dataSource; private readonly PostgresVectorStoreOptions _options; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// /// Initializes a new instance of the class. /// @@ -27,6 +32,14 @@ public PostgresVectorStore(NpgsqlDataSource dataSource, PostgresVectorStoreOptio this._dataSource = dataSource; this._options = options ?? new PostgresVectorStoreOptions(); this._postgresClient = new PostgresVectorStoreDbClient(this._dataSource, this._options.Schema); + + var connectionStringBuilder = new NpgsqlConnectionStringBuilder(dataSource.ConnectionString); + + this._metadata = new() + { + VectorStoreName = "postgresql", + DatabaseName = connectionStringBuilder.Database + }; } /// @@ -38,6 +51,14 @@ internal PostgresVectorStore(IPostgresVectorStoreDbClient postgresDbClient, Post { this._postgresClient = postgresDbClient; this._options = options ?? new PostgresVectorStoreOptions(); + + var connectionStringBuilder = new NpgsqlConnectionStringBuilder(postgresDbClient.DataSource.ConnectionString); + + this._metadata = new() + { + VectorStoreName = "postgresql", + DatabaseName = connectionStringBuilder.Database + }; } /// @@ -74,4 +95,17 @@ public virtual IVectorStoreRecordCollection GetCollection ?? throw new InvalidOperationException("Failed to cast record collection."); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(NpgsqlDataSource) ? this._dataSource : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs index bfac788a7cfd..32a2c85df7d9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Qdrant vector store. /// @@ -18,8 +20,8 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; /// public class QdrantVectorStore : IVectorStore { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Qdrant"; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; /// Qdrant client that can be used to manage the collections and points in a Qdrant store. private readonly MockableQdrantClient _qdrantClient; @@ -48,6 +50,11 @@ internal QdrantVectorStore(MockableQdrantClient qdrantClient, QdrantVectorStoreO this._qdrantClient = qdrantClient; this._options = options ?? new QdrantVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreName = "qdrant" + }; } /// @@ -88,7 +95,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._metadata.VectorStoreName, OperationName = "ListCollections" }; } @@ -98,4 +105,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat yield return collection; } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(QdrantClient) ? this._qdrantClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs index 4966917d3990..17137f87e05d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Redis vector store. /// @@ -18,8 +20,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; /// public class RedisVectorStore : IVectorStore { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Redis"; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; /// The redis database to read/write indices from. private readonly IDatabase _database; @@ -38,6 +40,12 @@ public RedisVectorStore(IDatabase database, RedisVectorStoreOptions? options = d this._database = database; this._options = options ?? new RedisVectorStoreOptions(); + + this._metadata = new() + { + VectorStoreName = "redis", + DatabaseName = database.Database.ToString() + }; } /// @@ -82,7 +90,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._metadata.VectorStoreName, OperationName = OperationName }; } @@ -96,4 +104,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(IDatabase) ? this._metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs index d9481ffc467d..772a28626b6e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; @@ -8,6 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.SqlServer; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// An implementation of backed by a SQL Server or Azure SQL database. /// @@ -16,6 +19,9 @@ public sealed class SqlServerVectorStore : IVectorStore private readonly string _connectionString; private readonly SqlServerVectorStoreOptions _options; + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// /// Initializes a new instance of the class. /// @@ -31,6 +37,14 @@ public SqlServerVectorStore(string connectionString, SqlServerVectorStoreOptions this._options = options is not null ? new() { Schema = options.Schema } : SqlServerVectorStoreOptions.Defaults; + + var connectionStringBuilder = new SqlConnectionStringBuilder(connectionString); + + this._metadata = new() + { + VectorStoreName = "microsoft.sql_server", + DatabaseName = connectionStringBuilder.InitialCatalog + }; } /// @@ -63,4 +77,16 @@ public async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancel yield return reader.GetString(reader.GetOrdinal("table_name")); } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs index 43b1a29b52d2..909b0a833bd0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs @@ -10,6 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.Sqlite; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a SQLite vector store. /// @@ -18,6 +20,9 @@ namespace Microsoft.SemanticKernel.Connectors.Sqlite; /// public class SqliteVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that will be used to manage the data in SQLite. private readonly DbConnection _connection; @@ -37,6 +42,12 @@ public SqliteVectorStore( this._connection = connection; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreName = "sqlite", + DatabaseName = connection.Database + }; } /// @@ -89,4 +100,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat yield return reader.GetString(ordinal); } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(DbConnection) ? this._connection : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs index 1c45d1e3ac65..34c4ddfe3b80 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; +#pragma warning disable SKEXP0020 // VectorStoreMetadata is experimental + /// /// Class for accessing the list of collections in a Weaviate vector store. /// @@ -19,6 +21,9 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; /// public class WeaviateVectorStore : IVectorStore { + /// Metadata about vector store. + private readonly VectorStoreMetadata _metadata; + /// that is used to interact with Weaviate API. private readonly HttpClient _httpClient; @@ -40,6 +45,11 @@ public WeaviateVectorStore(HttpClient httpClient, WeaviateVectorStoreOptions? op this._httpClient = httpClient; this._options = options ?? new(); + + this._metadata = new() + { + VectorStoreName = "weaviate" + }; } /// @@ -92,4 +102,17 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat } } } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreMetadata) ? this._metadata : + serviceType == typeof(HttpClient) ? this._httpClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj b/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj index f1dc235aa5bd..91b21a1f562c 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj @@ -42,5 +42,6 @@ Microsoft.Extensions.VectorData.IVectorStoreRecordCollection<TKey, TRecord> + diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs new file mode 100644 index 000000000000..0638874aaac7 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class KeywordHybridSearchMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems. + /// . + /// + public string? VectorStoreName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } + + /// + /// The name of a collection (table, container) within the database. + /// + public string? CollectionName { get; init; } +} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs new file mode 100644 index 000000000000..f9cc252db301 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class VectorizableTextSearchMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems. + /// . + /// + public string? VectorStoreName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } + + /// + /// The name of a collection (table, container) within the database. + /// + public string? CollectionName { get; init; } +} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs new file mode 100644 index 000000000000..b8cf96adb628 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class VectorizedSearchMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems. + /// . + /// + public string? VectorStoreName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } + + /// + /// The name of a collection (table, container) within the database. + /// + public string? CollectionName { get; init; } +} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs index 007dcf79da03..e90807f09f55 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; @@ -37,4 +38,16 @@ IVectorStoreRecordCollection GetCollection(string /// The to monitor for cancellation requests. The default is . /// The list of names of all the collections in the vector store. IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs new file mode 100644 index 000000000000..dd7b6b8d49a0 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class VectorStoreMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems. + /// . + /// + public string? VectorStoreName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } +} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs new file mode 100644 index 000000000000..15dee1549750 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs @@ -0,0 +1,28 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData; + +/// Provides metadata about an . +[Experimental("SKEXP0020")] +public class VectorStoreRecordCollectionMetadata +{ + /// The name of the vector store. + /// + /// Where possible, this maps to the appropriate name defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems. + /// . + /// + public string? VectorStoreName { get; init; } + + /// + /// The name of the database. + /// + public string? DatabaseName { get; init; } + + /// + /// The name of a collection (table, container) within the database. + /// + public string? CollectionName { get; init; } +} From a718acdc167e4bb8d5b8e0506370ad625a03598e Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 16:59:32 -0700 Subject: [PATCH 02/19] Added metadata and GetService for IVectorStoreRecordCollection --- .../TextEmbeddingVectorStore.cs | 2 +- ...extEmbeddingVectorStoreRecordCollection.cs | 11 ++++ .../Concepts/Search/VectorStore_TextSearch.cs | 10 +++ ...zureAISearchVectorStoreRecordCollection.cs | 61 +++++++++++++++---- ...mosDBMongoDBVectorStoreRecordCollection.cs | 49 ++++++++++++--- ...osmosDBNoSQLVectorStoreRecordCollection.cs | 45 ++++++++++++-- .../InMemoryVectorStoreRecordCollection.cs | 30 +++++++++ .../MongoDBVectorStoreRecordCollection.cs | 56 +++++++++++++---- .../PineconeVectorStoreRecordCollection.cs | 43 ++++++++++--- .../PostgresConstants.cs | 3 - .../PostgresVectorStore.cs | 3 +- .../PostgresVectorStoreRecordCollection.cs | 50 ++++++++++++--- .../PostgresVectorStoreUtils.cs | 20 ++++-- .../QdrantVectorStore.cs | 2 +- .../QdrantVectorStoreRecordCollection.cs | 56 +++++++++++++---- ...RedisHashSetVectorStoreRecordCollection.cs | 44 ++++++++++--- .../RedisJsonVectorStoreRecordCollection.cs | 48 ++++++++++++--- .../SqlServerVectorStoreRecordCollection.cs | 32 ++++++++++ .../SqliteVectorStoreRecordCollection.cs | 40 ++++++++++-- .../WeaviateVectorStoreRecordCollection.cs | 46 +++++++++++--- .../VectorSearch/IKeywordHybridSearch.cs | 15 +++++ .../VectorSearch/IVectorizableTextSearch.cs | 15 +++++ .../VectorSearch/IVectorizedSearch.cs | 15 +++++ .../KeywordHybridSearchMetadata.cs | 14 +++++ .../VectorizableTextSearchMetadata.cs | 14 +++++ .../VectorSearch/VectorizedSearchMetadata.cs | 14 +++++ .../VectorStorage/IVectorStore.cs | 2 + .../IVectorStoreRecordCollection.cs | 2 + .../Data/BaseVectorStoreTextSearchTests.cs | 10 +++ .../Search/MockVectorizableTextSearch.cs | 10 +++ .../Data/VectorStoreTextSearchTestBase.cs | 10 +++ .../Collections/CollectionConformanceTests.cs | 27 +++++++- 32 files changed, 689 insertions(+), 110 deletions(-) diff --git a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs index dba6e77f398b..4fd62592adf3 100644 --- a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs +++ b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStore.cs @@ -41,7 +41,7 @@ public IVectorStoreRecordCollection GetCollection( } /// - public virtual object? GetService(Type serviceType, object? serviceKey = null) + public object? GetService(Type serviceType, object? serviceKey = null) { ArgumentNullException.ThrowIfNull(serviceType); diff --git a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs index 000cb1ebba07..7ac4352e401f 100644 --- a/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs +++ b/dotnet/samples/Concepts/Memory/VectorStoreEmbeddingGeneration/TextEmbeddingVectorStoreRecordCollection.cs @@ -47,6 +47,7 @@ public TextEmbeddingVectorStoreRecordCollection(IVectorStoreRecordCollection + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] public string CollectionName => this._decoratedVectorStoreRecordCollection.CollectionName; /// @@ -132,6 +133,16 @@ public async Task> VectorizableTextSearchAsync(stri return await this.VectorizedSearchAsync(embeddingValue, options, cancellationToken).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._decoratedVectorStoreRecordCollection.GetService(serviceType, serviceKey); + } + /// /// Generate and add embeddings for each embedding field that has a on the provided record. /// diff --git a/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs b/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs index f6a3d4ab6356..c9cf9e4310ed 100644 --- a/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs +++ b/dotnet/samples/Concepts/Search/VectorStore_TextSearch.cs @@ -150,6 +150,16 @@ public async Task> VectorizableTextSearchAsync(stri return await vectorizedSearch.VectorizedSearchAsync(vectorizedQuery, options, cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + vectorizedSearch.GetService(serviceType, serviceKey); + } } /// diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs index eda71258ef24..97d1af5bd6fc 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs @@ -18,6 +18,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Azure AI Search as the underlying storage. /// @@ -29,8 +31,17 @@ public class AzureAISearchVectorStoreRecordCollection : IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureAISearch"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + + /// Metadata about vectorizable text search. + private readonly VectorizableTextSearchMetadata _vectorizableTextSearchMetadata; + + /// Metadata about keyword hybrid search. + private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -141,6 +152,17 @@ public AzureAISearchVectorStoreRecordCollection(SearchIndexClient searchIndexCli { this._mapper = new AzureAISearchGenericDataModelMapper(this._propertyReader.RecordDefinition) as IVectorStoreRecordMapper; } + + this._collectionMetadata = new() + { + VectorStoreName = "azure.aisearch", + DatabaseName = searchIndexClient.ServiceName, + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); + this._vectorizableTextSearchMetadata = VectorizableTextSearchMetadata.From(this._collectionMetadata); + this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -162,8 +184,8 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - CollectionName = this._collectionName, + VectorStoreType = this._collectionMetadata.VectorStoreName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = "GetIndex" }; } @@ -488,6 +510,23 @@ public Task> HybridSearchAsync(TVector vec return this.SearchAndMapToDataModelAsync(keywordsCombined, searchOptions, internalOptions.IncludeVectors, cancellationToken); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(VectorizableTextSearchMetadata) ? this._vectorizableTextSearchMetadata : + serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : + serviceType == typeof(SearchIndexClient) ? this._searchIndexClient : + serviceType == typeof(SearchClient) ? this._searchClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Get the document with the given key and map it to the data model using the configured mapper type. /// @@ -517,7 +556,7 @@ public Task> HybridSearchAsync(TVector vec } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, OperationName, () => this._mapper!.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); @@ -580,7 +619,7 @@ private Task> MapToStorageModelAndUploadDocumentA if (this._mapper is not null) { var jsonObjects = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, OperationName, () => records.Select(this._mapper!.MapFromDataToStorageModel)); @@ -608,7 +647,7 @@ private async IAsyncEnumerable> MapSearchResultsAsyn await foreach (var result in results.ConfigureAwait(false)) { var document = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, operationName, () => this._options.JsonObjectCustomMapper!.MapFromStorageToDataModel(result.Document, new() { IncludeVectors = includeVectors })); @@ -688,8 +727,8 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - CollectionName = this._collectionName, + VectorStoreType = this._collectionMetadata.VectorStoreName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = operationName }; } @@ -697,8 +736,8 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, - CollectionName = this._collectionName, + VectorStoreType = this._collectionMetadata.VectorStoreName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = operationName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs index 13e31475447d..e060883cc5f0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs @@ -17,6 +17,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Azure CosmosDB MongoDB as the underlying storage. /// @@ -25,8 +27,11 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBMongoDB; public class AzureCosmosDBMongoDBVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureCosmosDBMongoDB"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; /// Property name to be used for search similarity score value. private const string ScorePropertyName = "similarityScore"; @@ -93,6 +98,15 @@ public AzureCosmosDBMongoDBVectorStoreRecordCollection( this._vectorStoragePropertyNames = this._propertyReader.VectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); this._mapper = this.InitializeMapper(); + + this._collectionMetadata = new() + { + VectorStoreName = "azure.cosmosdbmongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -108,8 +122,8 @@ public virtual async Task CreateCollectionAsync(CancellationToken cancellationTo { throw new VectorStoreOperationException("Collection already exists.") { - VectorStoreType = DatabaseName, - CollectionName = this.CollectionName, + VectorStoreType = this._collectionMetadata.VectorStoreName, + CollectionName = this._collectionMetadata.CollectionName, OperationName = "CreateCollection" }; } @@ -173,7 +187,7 @@ public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new() { IncludeVectors = includeVectors })); @@ -200,7 +214,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( if (record is not null) { yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new())); @@ -218,7 +232,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancel var replaceOptions = new ReplaceOptions { IsUpsert = true }; var storageModel = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -324,6 +338,21 @@ public virtual async Task> VectorizedSearchAsync(this.EnumerateAndMapSearchResultsAsync(cursor, searchOptions, cancellationToken)); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType == typeof(IMongoCollection) ? this._mongoCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async Task CreateIndexesAsync(string collectionName, CancellationToken cancellationToken) @@ -399,7 +428,7 @@ private async IAsyncEnumerable> EnumerateAndMapSearc { var score = response[ScorePropertyName].AsDouble; var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, new())); @@ -438,7 +467,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -455,7 +484,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs index dd0e245c8004..c7c3aac1b3b8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs @@ -17,6 +17,8 @@ namespace Microsoft.SemanticKernel.Connectors.AzureCosmosDBNoSQL; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Azure CosmosDB NoSQL as the underlying storage. /// @@ -28,8 +30,14 @@ public class AzureCosmosDBNoSQLVectorStoreRecordCollection : IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "AzureCosmosDBNoSQL"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + + /// Metadata about keyword hybrid search. + private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; /// A of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -164,6 +172,16 @@ public AzureCosmosDBNoSQLVectorStoreRecordCollection( .Concat([this._propertyReader.KeyProperty]) .Select(x => this._storagePropertyNames[x.DataModelPropertyName]) .ToList(); + + this._collectionMetadata = new() + { + VectorStoreName = "azure.cosmosdbnosql", + DatabaseName = database.Id, + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); + this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -442,6 +460,21 @@ public Task> HybridSearchAsync(TVector vec #endregion + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : + serviceType == typeof(Database) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private void VerifyVectorType(TVector? vector) @@ -468,7 +501,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -655,7 +688,7 @@ private async IAsyncEnumerable InternalGetAsync( await foreach (var jsonObject in this.GetItemsAsync(queryDefinition, cancellationToken).ConfigureAwait(false)) { yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); @@ -671,7 +704,7 @@ private async Task InternalUpsertAsync( const string OperationName = "UpsertItem"; var jsonObject = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -751,7 +784,7 @@ private async IAsyncEnumerable> MapSearchResultsAsyn jsonObject.Remove(scorePropertyName); var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs index 6fbcdf2633bf..38f9e302828d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs @@ -12,6 +12,8 @@ namespace Microsoft.SemanticKernel.Connectors.InMemory; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses an in memory dictionary as the underlying storage. /// @@ -22,6 +24,12 @@ public sealed class InMemoryVectorStoreRecordCollection : IVector #pragma warning restore CA1711 // Identifiers should not have incorrect suffix where TKey : notnull { + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + /// A set of types that vectors on the provided model may have. private static readonly HashSet s_supportedVectorTypes = [ @@ -81,6 +89,14 @@ public InMemoryVectorStoreRecordCollection(string collectionName, InMemoryVector // Assign resolvers. this._vectorResolver = CreateVectorResolver(this._options.VectorResolver, this._vectorProperties); this._keyResolver = CreateKeyResolver(this._options.KeyResolver, this._propertyReader.KeyProperty); + + this._collectionMetadata = new() + { + VectorStoreName = "inmemory", + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -278,6 +294,20 @@ public async Task> VectorizedSearchAsync(T return new VectorSearchResults(vectorSearchResultList) { TotalCount = count }; } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(ConcurrentDictionary>) ? this._internalCollections : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Get the collection dictionary from the internal storage, throws if it does not exist. /// diff --git a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs index dc2aa163a803..af9df96dc78e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs @@ -16,6 +16,8 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses MongoDB as the underlying storage. /// @@ -24,8 +26,14 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; public class MongoDBVectorStoreRecordCollection : IVectorStoreRecordCollection, IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "MongoDB"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + + /// Metadata about keyword hybrid search. + private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; /// Property name to be used for search similarity score value. private const string ScorePropertyName = "similarityScore"; @@ -95,6 +103,16 @@ public MongoDBVectorStoreRecordCollection( this._vectorStoragePropertyNames = this._propertyReader.VectorProperties.Select(property => this._storagePropertyNames[property.DataModelPropertyName]).ToList(); this._mapper = this.InitializeMapper(); + + this._collectionMetadata = new() + { + VectorStoreName = "mongodb", + DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); + this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -110,7 +128,7 @@ public virtual async Task CreateCollectionAsync(CancellationToken cancellationTo { throw new VectorStoreOperationException("Collection already exists.") { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = "CreateCollection" }; @@ -181,7 +199,7 @@ public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new() { IncludeVectors = includeVectors })); @@ -208,7 +226,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( if (record is not null) { yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new())); @@ -226,7 +244,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancel var replaceOptions = new ReplaceOptions { IsUpsert = true }; var storageModel = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -373,6 +391,22 @@ public async Task> HybridSearchAsync(TVect cancellationToken).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : + serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : + serviceType == typeof(IMongoCollection) ? this._mongoCollection : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async Task CreateIndexesAsync(string collectionName, CancellationToken cancellationToken) @@ -490,7 +524,7 @@ private async IAsyncEnumerable> EnumerateAndMapSearc { var score = response[ScorePropertyName].AsDouble; var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, new() { IncludeVectors = includeVectors })); @@ -529,7 +563,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -546,7 +580,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -577,7 +611,7 @@ private async Task RunOperationWithRetryAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -611,7 +645,7 @@ private async Task RunOperationWithRetryAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 6e44feda9334..63dc74146125 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -14,6 +14,8 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Pinecone as the underlying storage. /// @@ -22,7 +24,6 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - private const string DatabaseName = "Pinecone"; private const string CreateCollectionName = "CreateCollection"; private const string CollectionExistsName = "CollectionExists"; private const string DeleteCollectionName = "DeleteCollection"; @@ -34,6 +35,12 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreRecordCollectionOptions _options; private readonly VectorStoreRecordPropertyReader _propertyReader; @@ -87,6 +94,14 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st // Default Mapper. this._mapper = new PineconeVectorStoreRecordMapper(this._propertyReader); } + + this._collectionMetadata = new() + { + VectorStoreName = "pinecone", + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -165,7 +180,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( () => index.Fetch(keys, indexNamespace, cancellationToken)).ConfigureAwait(false); var records = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, GetOperationName, () => results.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); @@ -208,7 +223,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); var vector = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, UpsertOperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -230,7 +245,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -288,7 +303,7 @@ public virtual async Task> VectorizedSearchAsync @@ -313,6 +328,20 @@ public virtual async Task> VectorizedSearchAsync(records.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(Sdk.PineconeClient) ? this._pineconeClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + private async Task RunOperationAsync(string operationName, Func> operation) { try @@ -323,7 +352,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -340,7 +369,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs index f8784890e83a..ef2491a13b48 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresConstants.cs @@ -8,9 +8,6 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres; internal static class PostgresConstants { - /// The name of this database for telemetry purposes. - public const string DatabaseName = "Postgres"; - /// A of types that a key on the provided model may have. public static readonly HashSet SupportedKeyTypes = [ diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs index ba52e03e22b3..5fc5afc5c3a7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs @@ -67,7 +67,8 @@ public virtual IAsyncEnumerable ListCollectionNamesAsync(CancellationTok const string OperationName = "ListCollectionNames"; return PostgresVectorStoreUtils.WrapAsyncEnumerableAsync( this._postgresClient.GetTablesAsync(cancellationToken), - OperationName + OperationName, + vectorStoreName: this._metadata.VectorStoreName ); } diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs index ce619398bf99..bc6cdb721db4 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.Postgres; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Represents a collection of vector store records in a Postgres database. /// @@ -24,6 +26,12 @@ public class PostgresVectorStoreRecordCollection : IVectorStoreRe /// public string CollectionName { get; } + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + /// Postgres client that is used to interact with the database. private readonly IPostgresVectorStoreDbClient _client; @@ -102,6 +110,17 @@ internal PostgresVectorStoreRecordCollection(IPostgresVectorStoreDbClient client { this._mapper = new PostgresVectorStoreRecordMapper(this._propertyReader); } + + var connectionStringBuilder = new NpgsqlConnectionStringBuilder(client.DataSource.ConnectionString); + + this._collectionMetadata = new() + { + VectorStoreName = "postgresql", + DatabaseName = connectionStringBuilder.Database, + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -146,7 +165,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancella const string OperationName = "Upsert"; var storageModel = VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -173,7 +192,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -207,7 +226,7 @@ await this.RunOperationAsync(OperationName, () => if (row is null) { return default; } return VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(row, new() { IncludeVectors = includeVectors })); @@ -227,14 +246,15 @@ public virtual IAsyncEnumerable GetBatchAsync(IEnumerable keys, G this._client.GetBatchAsync(this.CollectionName, keys, this._propertyReader.RecordDefinition.Properties, includeVectors, cancellationToken) .SelectAsync(row => VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(row, new() { IncludeVectors = includeVectors })), cancellationToken ), OperationName, - this.CollectionName + this.CollectionName, + this._collectionMetadata.VectorStoreName ); } @@ -303,7 +323,7 @@ public virtual Task> VectorizedSearchAsync .SelectAsync(result => { var record = VectorStoreErrorHandler.RunModelConversion( - PostgresConstants.DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel( @@ -317,6 +337,20 @@ public virtual Task> VectorizedSearchAsync }); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(NpgsqlDataSource) ? this._client.DataSource : + serviceType.IsInstanceOfType(this) ? this : + null; + } + private Task InternalCreateCollectionAsync(bool ifNotExists, CancellationToken cancellationToken = default) { return this._client.CreateTableAsync(this.CollectionName, this._propertyReader.RecordDefinition.Properties, ifNotExists, cancellationToken); @@ -332,7 +366,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = PostgresConstants.DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -349,7 +383,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = PostgresConstants.DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs index 27fa7181bdc5..eddcef924d42 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs @@ -18,16 +18,21 @@ internal static class PostgresVectorStoreUtils /// The async enumerable to wrap. /// The name of the operation being performed. /// The name of the collection being operated on. + /// The name of the vector store being operated on. /// An async enumerable that will throw a if an exception is thrown while iterating over the original enumerator. - public static async IAsyncEnumerable WrapAsyncEnumerableAsync(IAsyncEnumerable asyncEnumerable, string operationName, string? collectionName = null) + public static async IAsyncEnumerable WrapAsyncEnumerableAsync( + IAsyncEnumerable asyncEnumerable, + string operationName, + string? collectionName = null, + string? vectorStoreName = null) { var enumerator = asyncEnumerable.ConfigureAwait(false).GetAsyncEnumerator(); - var nextResult = await GetNextAsync(enumerator, operationName, collectionName).ConfigureAwait(false); + var nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreName).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.item; - nextResult = await GetNextAsync(enumerator, operationName, collectionName).ConfigureAwait(false); + nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreName).ConfigureAwait(false); } } @@ -38,8 +43,13 @@ public static async IAsyncEnumerable WrapAsyncEnumerableAsync(IAsyncEnumer /// The enumerator to get the next result from. /// The name of the operation being performed. /// The name of the collection being operated on. + /// The name of the vector store being operated on. /// A value indicating whether there are more results and the current string if true. - public static async Task<(T item, bool more)> GetNextAsync(ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, string operationName, string? collectionName = null) + public static async Task<(T item, bool more)> GetNextAsync( + ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, + string operationName, + string? collectionName = null, + string? vectorStoreName = null) { try { @@ -50,7 +60,7 @@ public static async IAsyncEnumerable WrapAsyncEnumerableAsync(IAsyncEnumer { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = PostgresConstants.DatabaseName, + VectorStoreType = vectorStoreName, CollectionName = collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs index 32a2c85df7d9..cf95de6e0f58 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs @@ -114,7 +114,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat return serviceKey is not null ? null : serviceType == typeof(VectorStoreMetadata) ? this._metadata : - serviceType == typeof(QdrantClient) ? this._qdrantClient : + serviceType == typeof(QdrantClient) ? this._qdrantClient.QdrantClient : serviceType.IsInstanceOfType(this) ? this : null; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs index 760aeaae24f4..b09c3f3f0f50 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs @@ -14,6 +14,8 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Qdrant as the underlying storage. /// @@ -25,6 +27,15 @@ public class QdrantVectorStoreRecordCollection : IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + + /// Metadata about keyword hybrid search. + private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; + /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -38,9 +49,6 @@ public class QdrantVectorStoreRecordCollection : /// The default options for hybrid vector search. private static readonly HybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new(); - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Qdrant"; - /// The name of the upsert operation for telemetry purposes. private const string UpsertName = "Upsert"; @@ -128,6 +136,15 @@ internal QdrantVectorStoreRecordCollection(MockableQdrantClient qdrantClient, st this._propertyReader, this._options.HasNamedVectors); } + + this._collectionMetadata = new() + { + VectorStoreName = "qdrant", + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); + this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -352,7 +369,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken c // Create point from record. var pointStruct = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, UpsertName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -371,7 +388,7 @@ async Task IVectorStoreRecordCollection.UpsertAsync(TRecord // Create point from record. var pointStruct = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, UpsertName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -390,7 +407,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -413,7 +430,7 @@ async IAsyncEnumerable IVectorStoreRecordCollection.UpsertB // Create points from records. var pointStructs = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, UpsertName, () => records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -471,7 +488,7 @@ private async IAsyncEnumerable GetBatchByPointIdAsync( } yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(pointStruct, new() { IncludeVectors = includeVectors })); @@ -532,7 +549,7 @@ public virtual async Task> VectorizedSearchAsync> HybridSearchAsync(TVect point, this._mapper, internalOptions.IncludeVectors, - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, "Query")); return new VectorSearchResults(mappedResults.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : + serviceType == typeof(QdrantClient) ? this._qdrantClient.QdrantClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Run the given operation and wrap any with ."/> /// @@ -647,7 +679,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = operationName }; @@ -671,7 +703,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs index a08fe1e86628..a1e0f879780b 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs @@ -14,6 +14,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Redis HashSets as the underlying storage. /// @@ -22,8 +24,11 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; public class RedisHashSetVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Redis"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -141,6 +146,15 @@ public RedisHashSetVectorStoreRecordCollection(IDatabase database, string collec // Default Mapper. this._mapper = new RedisHashSetVectorStoreRecordMapper(this._propertyReader); } + + this._collectionMetadata = new() + { + VectorStoreName = "redis", + DatabaseName = database.Database.ToString(), + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -162,7 +176,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = "FT.INFO" }; @@ -251,7 +265,7 @@ await this.RunOperationAsync("FT.DROPINDEX", // Convert to the caller's data model. return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, operationName, () => @@ -309,7 +323,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken // Map. var redisHashSetRecord = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, "HSET", () => this._mapper.MapFromDataToStorageModel(record)); @@ -377,7 +391,7 @@ public virtual async Task> VectorizedSearchAsync @@ -396,6 +410,20 @@ public virtual async Task> VectorizedSearchAsync(mappedResults.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(IDatabase) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Prefix the key with the collection name if the option is set. /// @@ -445,7 +473,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = operationName }; @@ -468,7 +496,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs index af6a0a7d220f..91dd5a9adba7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs @@ -17,6 +17,8 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Redis JSON as the underlying storage. /// @@ -25,8 +27,11 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; public class RedisJsonVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Redis"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -123,6 +128,15 @@ public RedisJsonVectorStoreRecordCollection(IDatabase database, string collectio // Default Mapper. this._mapper = new RedisJsonVectorStoreRecordMapper(this._propertyReader.KeyPropertyJsonName, this._jsonSerializerOptions); } + + this._collectionMetadata = new() + { + VectorStoreName = "redis", + DatabaseName = database.Database.ToString(), + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -144,7 +158,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = "FT.INFO" }; @@ -233,7 +247,7 @@ await this.RunOperationAsync("FT.DROPINDEX", // Convert to the caller's data model. return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, "GET", () => @@ -282,7 +296,7 @@ public virtual async IAsyncEnumerable GetBatchAsync(IEnumerable // Convert to the caller's data model. yield return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, "MGET", () => @@ -326,7 +340,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken // Map. var redisJsonRecord = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this._collectionName, "SET", () => @@ -360,7 +374,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable @@ -416,7 +430,7 @@ public virtual async Task> VectorizedSearchAsync @@ -438,6 +452,20 @@ public virtual async Task> VectorizedSearchAsync(mappedResults.ToAsyncEnumerable()); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(IDatabase) ? this._database : + serviceType.IsInstanceOfType(this) ? this : + null; + } + /// /// Prefix the key with the collection name if the option is set. /// @@ -486,7 +514,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = operationName }; @@ -510,7 +538,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs index f68318a92836..f80ae5f3c794 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.SqlServer; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// An implementation of backed by a SQL Server or Azure SQL database. /// @@ -19,6 +21,12 @@ public sealed class SqlServerVectorStoreRecordCollection #pragma warning restore CA1711 : IVectorStoreRecordCollection where TKey : notnull { + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); private static readonly SqlServerVectorStoreRecordCollectionOptions s_defaultOptions = new(); @@ -88,6 +96,17 @@ public SqlServerVectorStoreRecordCollection( this._mapper = new RecordMapper(propertyReader); } + + var connectionStringBuilder = new SqlConnectionStringBuilder(connectionString); + + this._collectionMetadata = new() + { + VectorStoreName = "microsoft.sql_server", + DatabaseName = connectionStringBuilder.InitialCatalog, + CollectionName = name + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -358,6 +377,19 @@ public async Task> VectorizedSearchAsync(T }, cancellationToken, "VectorizedSearch", this.CollectionName).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType.IsInstanceOfType(this) ? this : + null; + } + private async IAsyncEnumerable> ReadVectorSearchResultsAsync( SqlConnection connection, SqlCommand command, diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs index 22ad3b67c403..85a05a62f8b1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs @@ -11,6 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.Sqlite; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses SQLite as the underlying storage. /// @@ -21,8 +23,11 @@ public class SqliteVectorStoreRecordCollection : IVectorStoreRecordCollection #pragma warning restore CA1711 // Identifiers should not have incorrect { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "SQLite"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; /// that will be used to manage the data in SQLite. private readonly DbConnection _connection; @@ -112,6 +117,15 @@ public SqliteVectorStoreRecordCollection( this._mapper = this.InitializeMapper(); this._commandBuilder = new SqliteVectorStoreCollectionCommandBuilder(this._connection); + + this._collectionMetadata = new() + { + VectorStoreName = "sqlite", + DatabaseName = connection.Database, + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -301,6 +315,20 @@ public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken #endregion + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(DbConnection) ? this._connection : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async IAsyncEnumerable> EnumerateAndMapSearchResultsAsync( @@ -501,7 +529,7 @@ private async Task InternalUpsertAsync(TRecord record, CancellationT const string OperationName = "Upsert"; var storageModel = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -524,7 +552,7 @@ private IAsyncEnumerable InternalUpsertBatchAsync(IEnumerable VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -651,7 +679,7 @@ private TRecord GetAndMapRecord( } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors })); @@ -667,7 +695,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 393b9a841cbb..b6a26e7db5c9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -16,6 +16,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; +#pragma warning disable SKEXP0020 // Metadata classes are experimental + /// /// Service for storing and retrieving vector records, that uses Weaviate as the underlying storage. /// @@ -24,8 +26,14 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; public class WeaviateVectorStoreRecordCollection : IVectorStoreRecordCollection, IKeywordHybridSearch #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { - /// The name of this database for telemetry purposes. - private const string DatabaseName = "Weaviate"; + /// Metadata about vector store record collection. + private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; + + /// Metadata about vectorized search. + private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; + + /// Metadata about keyword hybrid search. + private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = @@ -154,6 +162,15 @@ public WeaviateVectorStoreRecordCollection( // Assign mapper. this._mapper = this.InitializeMapper(); + + this._collectionMetadata = new() + { + VectorStoreName = "weaviate", + CollectionName = collectionName + }; + + this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); + this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -270,7 +287,7 @@ public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken c } return VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(jsonObject!, new() { IncludeVectors = includeVectors })); @@ -312,7 +329,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable { var jsonObjects = records.Select(record => VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -395,6 +412,21 @@ public async Task> HybridSearchAsync(TVect return await this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.HybridScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is not null ? null : + serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : + serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : + serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : + serviceType == typeof(HttpClient) ? this._httpClient : + serviceType.IsInstanceOfType(this) ? this : + null; + } + #region private private async Task> ExecuteQueryAsync(string query, bool includeVectors, string scorePropertyName, string operationName, CancellationToken cancellationToken) @@ -409,7 +441,7 @@ private async Task> ExecuteQueryAsync(string query, { throw new VectorStoreOperationException($"Error occurred during vector search. Response: {content}") { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -420,7 +452,7 @@ private async Task> ExecuteQueryAsync(string query, var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!, scorePropertyName); var record = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, + this._collectionMetadata.VectorStoreName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors })); @@ -483,7 +515,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = DatabaseName, + VectorStoreType = this._collectionMetadata.VectorStoreName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs index 53d2e062fcda..313abfecfeed 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -26,4 +28,17 @@ Task> HybridSearchAsync( ICollection keywords, HybridSearchOptions? options = default, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs index 5368c5301828..560a26e53748 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Diagnostics.CodeAnalysis; +using System; using System.Threading; using System.Threading.Tasks; @@ -22,4 +24,17 @@ Task> VectorizableTextSearchAsync( string searchText, VectorSearchOptions? options = default, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs index b2a5a54194a6..565420ac36c4 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs @@ -1,5 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Diagnostics.CodeAnalysis; +using System; using System.Threading; using System.Threading.Tasks; @@ -23,4 +25,17 @@ Task> VectorizedSearchAsync( TVector vector, VectorSearchOptions? options = default, CancellationToken cancellationToken = default); + + /// Asks the for an object of the specified type . + /// The type of object being requested. + /// An optional key that can be used to help identify the target service. + /// The found object, otherwise . + /// is . + /// + /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , + /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// may be used to request it. + /// + [Experimental("SKEXP0020")] + object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs index 0638874aaac7..42a5b8fad242 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs @@ -25,4 +25,18 @@ public class KeywordHybridSearchMetadata /// The name of a collection (table, container) within the database. /// public string? CollectionName { get; init; } + + /// + /// Initializes an instance of from . + /// + /// Instance of . + public static KeywordHybridSearchMetadata From(VectorStoreRecordCollectionMetadata collectionMetadata) + { + return new() + { + VectorStoreName = collectionMetadata.VectorStoreName, + DatabaseName = collectionMetadata.DatabaseName, + CollectionName = collectionMetadata.CollectionName + }; + } } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs index f9cc252db301..041baaf89f83 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs @@ -25,4 +25,18 @@ public class VectorizableTextSearchMetadata /// The name of a collection (table, container) within the database. /// public string? CollectionName { get; init; } + + /// + /// Initializes an instance of from . + /// + /// Instance of . + public static VectorizableTextSearchMetadata From(VectorStoreRecordCollectionMetadata collectionMetadata) + { + return new() + { + VectorStoreName = collectionMetadata.VectorStoreName, + DatabaseName = collectionMetadata.DatabaseName, + CollectionName = collectionMetadata.CollectionName + }; + } } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs index b8cf96adb628..110234eab081 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs @@ -25,4 +25,18 @@ public class VectorizedSearchMetadata /// The name of a collection (table, container) within the database. /// public string? CollectionName { get; init; } + + /// + /// Initializes an instance of from . + /// + /// Instance of . + public static VectorizedSearchMetadata From(VectorStoreRecordCollectionMetadata collectionMetadata) + { + return new() + { + VectorStoreName = collectionMetadata.VectorStoreName, + DatabaseName = collectionMetadata.DatabaseName, + CollectionName = collectionMetadata.CollectionName + }; + } } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs index e90807f09f55..e6341a2d8d3d 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStore.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Threading; namespace Microsoft.Extensions.VectorData; @@ -49,5 +50,6 @@ IVectorStoreRecordCollection GetCollection(string /// including itself or any services it might be wrapping. For example, to access the for the instance, /// may be used to request it. /// + [Experimental("SKEXP0020")] object? GetService(Type serviceType, object? serviceKey = null); } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs index b8e410d4afd5..780e535cf346 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/IVectorStoreRecordCollection.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -19,6 +20,7 @@ public interface IVectorStoreRecordCollection : IVectorizedSearch /// /// Gets the name of the collection. /// + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] string CollectionName { get; } /// diff --git a/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs b/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs index 143c61f69e5f..ea852dbb36d7 100644 --- a/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs +++ b/dotnet/src/IntegrationTests/Data/BaseVectorStoreTextSearchTests.cs @@ -108,6 +108,16 @@ public async Task> VectorizableTextSearchAsync(stri return await vectorizedSearch.VectorizedSearchAsync(vectorizedQuery, options, cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + vectorizedSearch.GetService(serviceType, serviceKey); + } } /// diff --git a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs index b39976adbebf..17609c979d78 100644 --- a/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs +++ b/dotnet/src/SemanticKernel.AotTests/UnitTests/Search/MockVectorizableTextSearch.cs @@ -18,6 +18,16 @@ public Task> VectorizableTextSearchAsync(string sea return Task.FromResult(new VectorSearchResults(this._searchResults)); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + null; + } + private static async IAsyncEnumerable> ToAsyncEnumerable(IEnumerable> searchResults) { foreach (var result in searchResults) diff --git a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs index c01fe06eddf4..9c56a71974ca 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Data/VectorStoreTextSearchTestBase.cs @@ -131,6 +131,16 @@ public async Task> VectorizableTextSearchAsync(stri var vectorizedQuery = await textEmbeddingGeneration!.GenerateEmbeddingAsync(searchText, cancellationToken: cancellationToken).ConfigureAwait(false); return await vectorizedSearch.VectorizedSearchAsync(vectorizedQuery, options, cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + ArgumentNullException.ThrowIfNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + vectorizedSearch.GetService(serviceType, serviceKey); + } } /// diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 16f8679df842..8f027d786c81 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -86,7 +86,14 @@ private async Task CreateCollection() try { Assert.True(await collection.CollectionExistsAsync()); - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; +#pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + Assert.NotNull(collectionMetadata?.CollectionName); + + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); } finally { @@ -103,7 +110,14 @@ private async Task CreateCollectionIfNotExistsMoreThanOnce() try { Assert.True(await collection.CollectionExistsAsync()); - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; +#pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + Assert.NotNull(collectionMetadata?.CollectionName); + + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); await collection.CreateCollectionIfNotExistsAsync(); } @@ -122,7 +136,14 @@ private async Task CreateCollectionMoreThanOnce() try { Assert.True(await collection.CollectionExistsAsync()); - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + +#pragma warning disable SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; +#pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. + + Assert.NotNull(collectionMetadata?.CollectionName); + + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); await collection.CreateCollectionIfNotExistsAsync(); From b5bc68c04e8c9bbbc80cb3eabd42a88672d31c8e Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:05:22 -0700 Subject: [PATCH 03/19] Small updates in tests --- .../Collections/CollectionConformanceTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 8f027d786c81..349259d51671 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -91,7 +91,9 @@ private async Task CreateCollection() var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; #pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - Assert.NotNull(collectionMetadata?.CollectionName); + Assert.NotNull(collectionMetadata); + Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.CollectionName); Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); } From c35ac3e5fa88ebf0d11a6afb58294683a7671ca2 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:10:04 -0700 Subject: [PATCH 04/19] Updated baseline version --- .../VectorData.Abstractions/VectorData.Abstractions.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj b/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj index 91b21a1f562c..50aec26b5951 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorData.Abstractions.csproj @@ -13,7 +13,7 @@ 9.0.0-preview.1.25161.1 9.0.0.0 - 9.0.0-preview.1.25078.1 + 9.0.0-preview.1.25161.1 Microsoft.Extensions.VectorData.Abstractions $(AssemblyName) Abstractions for vector database access. From 5844a653fb5d66787209c83eb5023fbdb4b0678b Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:29:04 -0700 Subject: [PATCH 05/19] Small updates --- .../AzureAISearchVectorStore.cs | 12 ++++++------ .../VectorSearch/IVectorizableTextSearch.cs | 2 +- .../VectorSearch/IVectorizedSearch.cs | 2 +- .../Collections/CollectionConformanceTests.cs | 11 +++++++++-- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs index 4d551c3cb1e7..ce0bb6e72386 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs @@ -83,11 +83,11 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat var indexNamesEnumerable = this._searchIndexClient.GetIndexNamesAsync(cancellationToken).ConfigureAwait(false); var indexNamesEnumerator = indexNamesEnumerable.GetAsyncEnumerator(); - var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata).ConfigureAwait(false); + var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreName).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.name; - nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata).ConfigureAwait(false); + nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreName).ConfigureAwait(false); } } @@ -110,11 +110,11 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat /// around a yield return. /// /// The enumerator to get the next result from. - /// Metadata about vector store. + /// The vector store name. /// A value indicating whether there are more results and the current string if true. private static async Task<(string name, bool more)> GetNextIndexNameAsync( ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, - VectorStoreMetadata metadata) + string? vectorStoreName) { const string OperationName = "GetIndexNames"; @@ -127,7 +127,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = metadata.VectorStoreName, + VectorStoreType = vectorStoreName, OperationName = OperationName }; } @@ -135,7 +135,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = metadata.VectorStoreName, + VectorStoreType = vectorStoreName, OperationName = OperationName }; } diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs index 560a26e53748..c1bb85b5090b 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Diagnostics.CodeAnalysis; using System; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs index 565420ac36c4..6b77dbb84afb 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Diagnostics.CodeAnalysis; using System; +using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 349259d51671..4efe7a7a80b8 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -93,6 +93,7 @@ private async Task CreateCollection() Assert.NotNull(collectionMetadata); Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.DatabaseName); Assert.NotNull(collectionMetadata.CollectionName); Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); @@ -117,7 +118,10 @@ private async Task CreateCollectionIfNotExistsMoreThanOnce() var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; #pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - Assert.NotNull(collectionMetadata?.CollectionName); + Assert.NotNull(collectionMetadata); + Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.DatabaseName); + Assert.NotNull(collectionMetadata.CollectionName); Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); @@ -143,7 +147,10 @@ private async Task CreateCollectionMoreThanOnce() var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; #pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - Assert.NotNull(collectionMetadata?.CollectionName); + Assert.NotNull(collectionMetadata); + Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.DatabaseName); + Assert.NotNull(collectionMetadata.CollectionName); Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); From cdbeeb58b8ff5e15d1c45870b65bc8f87eed35d7 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 17:46:19 -0700 Subject: [PATCH 06/19] Updated suppression files --- .../CompatibilitySuppressions.xml | 255 +----------------- 1 file changed, 12 insertions(+), 243 deletions(-) diff --git a/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml b/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml index cd9bfbaa3ca7..6281be710692 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml +++ b/dotnet/src/Connectors/VectorData.Abstractions/CompatibilitySuppressions.xml @@ -1,317 +1,86 @@  - - CP0001 - T:Microsoft.Extensions.VectorData.DeleteRecordOptions - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.UpsertRecordOptions - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.VectorSearchOptions - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.DeleteRecordOptions - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.UpsertRecordOptions - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.VectorSearchOptions - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.DeleteRecordOptions - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.UpsertRecordOptions - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0001 - T:Microsoft.Extensions.VectorData.VectorSearchOptions - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},Microsoft.Extensions.VectorData.DeleteRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0002 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},Microsoft.Extensions.VectorData.UpsertRecordOptions,System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IKeywordHybridSearch`1.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorStore.GetService(System.Type,System.Object) lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},System.Threading.CancellationToken) - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net462/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IKeywordHybridSearch`1.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorStore.GetService(System.Type,System.Object) lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},System.Threading.CancellationToken) - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/net8.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.VectorizableTextSearchAsync(System.String,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.VectorizedSearchAsync``1(``0,Microsoft.Extensions.VectorData.VectorSearchOptions{`0},System.Threading.CancellationToken) - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll - true - - - CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteAsync(`0,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IKeywordHybridSearch`1.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.DeleteBatchAsync(System.Collections.Generic.IEnumerable{`0},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizableTextSearch`1.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertAsync(`1,System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorizedSearch`1.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true CP0006 - M:Microsoft.Extensions.VectorData.IVectorStoreRecordCollection`2.UpsertBatchAsync(System.Collections.Generic.IEnumerable{`1},System.Threading.CancellationToken) + M:Microsoft.Extensions.VectorData.IVectorStore.GetService(System.Type,System.Object) lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll lib/netstandard2.0/Microsoft.Extensions.VectorData.Abstractions.dll true From dc1cba2ac656fb066a666437c132996120958d08 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:51:12 -0700 Subject: [PATCH 07/19] Fixed Azure AI Search unit tests --- .../AzureAISearchVectorStoreRecordCollectionTests.cs | 1 + .../AzureAISearchVectorStoreTests.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs index b786c8d8fa58..a9c787aeacdf 100644 --- a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreRecordCollectionTests.cs @@ -41,6 +41,7 @@ public AzureAISearchVectorStoreRecordCollectionTests() this._searchClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock.Setup(x => x.GetSearchClient(TestCollectionName)).Returns(this._searchClientMock.Object); + this._searchIndexClientMock.Setup(x => x.ServiceName).Returns("TestService"); } [Theory] diff --git a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs index b79b048a5f38..f4eed15172fb 100644 --- a/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.AzureAISearch.UnitTests/AzureAISearchVectorStoreTests.cs @@ -32,6 +32,7 @@ public AzureAISearchVectorStoreTests() this._searchClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock = new Mock(MockBehavior.Strict); this._searchIndexClientMock.Setup(x => x.GetSearchClient(TestCollectionName)).Returns(this._searchClientMock.Object); + this._searchIndexClientMock.Setup(x => x.ServiceName).Returns("TestService"); } [Fact] From f79ee7b34f4b3e420b451edd30c72d8fbc507cf8 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 19:02:13 -0700 Subject: [PATCH 08/19] Fixed unit tests for Sqlite --- .../Connectors.Sqlite.UnitTests/Fakes/FakeDBConnection.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/Connectors/Connectors.Sqlite.UnitTests/Fakes/FakeDBConnection.cs b/dotnet/src/Connectors/Connectors.Sqlite.UnitTests/Fakes/FakeDBConnection.cs index 7c318e1ef413..f46ae2a2c4a3 100644 --- a/dotnet/src/Connectors/Connectors.Sqlite.UnitTests/Fakes/FakeDBConnection.cs +++ b/dotnet/src/Connectors/Connectors.Sqlite.UnitTests/Fakes/FakeDBConnection.cs @@ -12,7 +12,7 @@ internal sealed class FakeDBConnection(DbCommand command) : DbConnection { public override string ConnectionString { get; set; } - public override string Database => throw new NotImplementedException(); + public override string Database => "TestDatabase"; public override string DataSource => throw new NotImplementedException(); From 648278fce40fe85af3fb8f252ffc7b1e1e578d63 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 18 Mar 2025 19:28:13 -0700 Subject: [PATCH 09/19] Fixed more unit tests --- .../IPostgresVectorStoreDbClient.cs | 5 +++++ .../Connectors.Memory.Postgres/PostgresVectorStore.cs | 8 ++------ .../PostgresVectorStoreDbClient.cs | 4 ++++ .../PostgresVectorStoreRecordCollection.cs | 4 +--- .../PostgresVectorStoreRecordCollectionTests.cs | 1 + .../PostgresVectorStoreTests.cs | 4 ++++ .../RedisHashSetVectorStoreRecordCollectionTests.cs | 1 + .../RedisJsonVectorStoreRecordCollectionTests.cs | 1 + .../Connectors.Redis.UnitTests/RedisVectorStoreTests.cs | 1 + 9 files changed, 20 insertions(+), 9 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs index 020aa46dbda6..9bb3d7e617d1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/IPostgresVectorStoreDbClient.cs @@ -21,6 +21,11 @@ internal interface IPostgresVectorStoreDbClient /// NpgsqlDataSource DataSource { get; } + /// + /// The name of the database. + /// + string? DatabaseName { get; } + /// /// Check if a table exists. /// diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs index 5fc5afc5c3a7..c011d17480e0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs @@ -33,12 +33,10 @@ public PostgresVectorStore(NpgsqlDataSource dataSource, PostgresVectorStoreOptio this._options = options ?? new PostgresVectorStoreOptions(); this._postgresClient = new PostgresVectorStoreDbClient(this._dataSource, this._options.Schema); - var connectionStringBuilder = new NpgsqlConnectionStringBuilder(dataSource.ConnectionString); - this._metadata = new() { VectorStoreName = "postgresql", - DatabaseName = connectionStringBuilder.Database + DatabaseName = this._postgresClient.DatabaseName }; } @@ -52,12 +50,10 @@ internal PostgresVectorStore(IPostgresVectorStoreDbClient postgresDbClient, Post this._postgresClient = postgresDbClient; this._options = options ?? new PostgresVectorStoreOptions(); - var connectionStringBuilder = new NpgsqlConnectionStringBuilder(postgresDbClient.DataSource.ConnectionString); - this._metadata = new() { VectorStoreName = "postgresql", - DatabaseName = connectionStringBuilder.Database + DatabaseName = this._postgresClient.DatabaseName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs index 07c228540038..a313306e311b 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreDbClient.cs @@ -26,10 +26,14 @@ internal class PostgresVectorStoreDbClient(NpgsqlDataSource dataSource, string s { private readonly string _schema = schema; + private readonly NpgsqlConnectionStringBuilder _connectionStringBuilder = new(dataSource.ConnectionString); + private IPostgresVectorStoreCollectionSqlBuilder _sqlBuilder = new PostgresVectorStoreCollectionSqlBuilder(); public NpgsqlDataSource DataSource { get; } = dataSource; + public string? DatabaseName => this._connectionStringBuilder.Database; + /// public async Task DoesTableExistsAsync(string tableName, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs index bc6cdb721db4..de7f495942ab 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs @@ -111,12 +111,10 @@ internal PostgresVectorStoreRecordCollection(IPostgresVectorStoreDbClient client this._mapper = new PostgresVectorStoreRecordMapper(this._propertyReader); } - var connectionStringBuilder = new NpgsqlConnectionStringBuilder(client.DataSource.ConnectionString); - this._collectionMetadata = new() { VectorStoreName = "postgresql", - DatabaseName = connectionStringBuilder.Database, + DatabaseName = this._client.DatabaseName, CollectionName = collectionName }; diff --git a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs index 0533ab28c3f3..3493162d5095 100644 --- a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreRecordCollectionTests.cs @@ -22,6 +22,7 @@ public class PostgresVectorStoreRecordCollectionTests public PostgresVectorStoreRecordCollectionTests() { this._postgresClientMock = new Mock(MockBehavior.Strict); + this._postgresClientMock.Setup(l => l.DatabaseName).Returns("TestDatabase"); } [Fact] diff --git a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs index 33cfc005a7bc..8a89582fc0f8 100644 --- a/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.Postgres.UnitTests/PostgresVectorStoreTests.cs @@ -26,6 +26,7 @@ public class PostgresVectorStoreTests public PostgresVectorStoreTests() { this._postgresClientMock = new Mock(MockBehavior.Strict); + this._postgresClientMock.Setup(l => l.DatabaseName).Returns("TestDatabase"); } [Fact] @@ -60,7 +61,10 @@ public void GetCollectionCallsFactoryIfProvided() var factoryMock = new Mock(MockBehavior.Strict); var collectionMock = new Mock>>(MockBehavior.Strict); var clientMock = new Mock(MockBehavior.Strict); + clientMock.Setup(x => x.DataSource).Returns(null); + clientMock.Setup(x => x.DatabaseName).Returns("TestDatabase"); + factoryMock .Setup(x => x.CreateVectorStoreRecordCollection>(It.IsAny(), TestCollectionName, null)) .Returns(collectionMock.Object); diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs index 117d3d1fcd4b..74c3dc49cf28 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisHashSetVectorStoreRecordCollectionTests.cs @@ -28,6 +28,7 @@ public class RedisHashSetVectorStoreRecordCollectionTests public RedisHashSetVectorStoreRecordCollectionTests() { this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); var batchMock = new Mock(); this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs index aa47dc512b8c..b729f1cba7a3 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordCollectionTests.cs @@ -32,6 +32,7 @@ public class RedisJsonVectorStoreRecordCollectionTests public RedisJsonVectorStoreRecordCollectionTests() { this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); var batchMock = new Mock(); this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs index baf2564c81a2..9280051fd266 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisVectorStoreTests.cs @@ -22,6 +22,7 @@ public class RedisVectorStoreTests public RedisVectorStoreTests() { this._redisDatabaseMock = new Mock(MockBehavior.Strict); + this._redisDatabaseMock.Setup(l => l.Database).Returns(0); var batchMock = new Mock(); this._redisDatabaseMock.Setup(x => x.CreateBatch(It.IsAny())).Returns(batchMock.Object); From ba81b70adb7419ee65add675890b4ee1bae4a266 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 19 Mar 2025 13:25:23 +0100 Subject: [PATCH 10/19] .Net MEVD: Split batches to account for the SQL Server 2100 parameter limit (#11047) fixes #11029 --- .../ExceptionWrapper.cs | 2 +- .../SqlServerCommandBuilder.cs | 30 +-- .../SqlServerConstants.cs | 3 + .../SqlServerVectorStoreRecordCollection.cs | 206 ++++++++++++++---- .../CRUD/SqlServerBatchConformanceTests.cs | 62 ++++++ .../SqlServerCommandBuilderTests.cs | 15 +- .../CRUD/BatchConformanceTests.cs | 2 +- 7 files changed, 249 insertions(+), 71 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/ExceptionWrapper.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/ExceptionWrapper.cs index 939e3530e797..6690f1d564a4 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/ExceptionWrapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/ExceptionWrapper.cs @@ -12,7 +12,7 @@ namespace Microsoft.SemanticKernel.Connectors.SqlServer; internal static class ExceptionWrapper { - private const string VectorStoreType = "SqlServer"; + internal const string VectorStoreType = "SqlServer"; internal static async Task WrapAsync( SqlConnection connection, diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerCommandBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerCommandBuilder.cs index 4fb9c32d4fe5..78d3ad51b998 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerCommandBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerCommandBuilder.cs @@ -155,16 +155,14 @@ internal static SqlCommand MergeIntoSingle( return command; } - internal static SqlCommand? MergeIntoMany( - SqlConnection connection, + internal static bool MergeIntoMany( + SqlCommand command, string? schema, string tableName, VectorStoreRecordKeyProperty keyProperty, IReadOnlyList properties, IEnumerable> records) { - SqlCommand command = connection.CreateCommand(); - StringBuilder sb = new(200); // The DECLARE statement creates a table variable to store the keys of the inserted rows. sb.AppendFormat("DECLARE @InsertedKeys TABLE (KeyColumn {0});", Map(keyProperty)); @@ -190,7 +188,7 @@ internal static SqlCommand MergeIntoSingle( if (rowIndex == 0) { - return null; // there is nothing to do! + return false; // there is nothing to do! } sb.Length -= (1 + Environment.NewLine.Length); // remove the last comma and newline @@ -225,7 +223,7 @@ internal static SqlCommand MergeIntoSingle( sb.Append("SELECT KeyColumn FROM @InsertedKeys;"); command.CommandText = sb.ToString(); - return command; + return true; } internal static SqlCommand DeleteSingle( @@ -246,12 +244,10 @@ internal static SqlCommand DeleteSingle( return command; } - internal static SqlCommand? DeleteMany( - SqlConnection connection, string? schema, string tableName, + internal static bool DeleteMany( + SqlCommand command, string? schema, string tableName, VectorStoreRecordKeyProperty keyProperty, IEnumerable keys) { - SqlCommand command = connection.CreateCommand(); - StringBuilder sb = new(100); sb.Append("DELETE FROM "); sb.AppendTableName(schema, tableName); @@ -261,11 +257,11 @@ internal static SqlCommand DeleteSingle( if (emptyKeys) { - return null; // there is nothing to do! + return false; } command.CommandText = sb.ToString(); - return command; + return true; } internal static SqlCommand SelectSingle( @@ -293,15 +289,13 @@ internal static SqlCommand SelectSingle( return command; } - internal static SqlCommand? SelectMany( - SqlConnection connection, string? schema, string tableName, + internal static bool SelectMany( + SqlCommand command, string? schema, string tableName, VectorStoreRecordKeyProperty keyProperty, IReadOnlyList properties, IEnumerable keys, bool includeVectors) { - SqlCommand command = connection.CreateCommand(); - StringBuilder sb = new(200); sb.AppendFormat("SELECT "); sb.AppendColumnNames(properties, includeVectors: includeVectors); @@ -315,11 +309,11 @@ internal static SqlCommand SelectSingle( if (emptyKeys) { - return null; // there is nothing to do! + return false; // there is nothing to do! } command.CommandText = sb.ToString(); - return command; + return true; } internal static SqlCommand SelectVector( diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerConstants.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerConstants.cs index 6b81cbac1ef6..d8ce0f1354e7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerConstants.cs @@ -7,6 +7,9 @@ namespace Microsoft.SemanticKernel.Connectors.SqlServer; internal static class SqlServerConstants { + // The actual number is actually higher (2_100), but we want to avoid any kind of "off by one" errors. + internal const int MaxParameterCount = 2_000; + internal static readonly HashSet SupportedKeyTypes = [ typeof(int), // INT diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs index f68318a92836..9b4ce3b29078 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs @@ -177,21 +177,65 @@ public async Task DeleteBatchAsync(IEnumerable keys, CancellationToken can Verify.NotNull(keys); using SqlConnection connection = new(this._connectionString); - using SqlCommand? command = SqlServerCommandBuilder.DeleteMany( - connection, - this._options.Schema, - this.CollectionName, - this._propertyReader.KeyProperty, - keys); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + + using SqlTransaction transaction = connection.BeginTransaction(); + int taken = 0; - if (command is null) + try { - return; // keys is empty, there is nothing to delete + while (true) + { +#if NET + SqlCommand command = new("", connection, transaction); + await using (command.ConfigureAwait(false)) +#else + using (SqlCommand command = new("", connection, transaction)) +#endif + { + if (!SqlServerCommandBuilder.DeleteMany( + command, + this._options.Schema, + this.CollectionName, + this._propertyReader.KeyProperty, + keys.Skip(taken).Take(SqlServerConstants.MaxParameterCount))) + { + break; // keys is empty, there is nothing to delete + } + + checked + { + taken += command.Parameters.Count; + } + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (taken > 0) + { +#if NET + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); +#else + transaction.Commit(); +#endif + } } + catch (Exception ex) + { +#if NET + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); +#else + transaction.Rollback(); +#endif - await ExceptionWrapper.WrapAsync(connection, command, - static (cmd, ct) => cmd.ExecuteNonQueryAsync(ct), - cancellationToken, "DeleteBatch", this.CollectionName).ConfigureAwait(false); + throw new VectorStoreOperationException(ex.Message, ex) + { + OperationName = "DeleteBatch", + VectorStoreType = ExceptionWrapper.VectorStoreType, + CollectionName = this.CollectionName + }; + } } /// @@ -235,30 +279,44 @@ public async IAsyncEnumerable GetBatchAsync(IEnumerable keys, Get bool includeVectors = options?.IncludeVectors is true; using SqlConnection connection = new(this._connectionString); - using SqlCommand? command = SqlServerCommandBuilder.SelectMany( - connection, - this._options.Schema, - this.CollectionName, - this._propertyReader.KeyProperty, - this._propertyReader.Properties, - keys, - includeVectors); + using SqlCommand command = connection.CreateCommand(); + int taken = 0; - if (command is null) + do { - yield break; // keys is empty - } + if (command.Parameters.Count > 0) + { + command.Parameters.Clear(); // We reuse the same command for the next batch. + } - using SqlDataReader reader = await ExceptionWrapper.WrapAsync(connection, command, - static (cmd, ct) => cmd.ExecuteReaderAsync(ct), - cancellationToken, "GetBatch", this.CollectionName).ConfigureAwait(false); + if (!SqlServerCommandBuilder.SelectMany( + command, + this._options.Schema, + this.CollectionName, + this._propertyReader.KeyProperty, + this._propertyReader.Properties, + keys.Skip(taken).Take(SqlServerConstants.MaxParameterCount), + includeVectors)) + { + yield break; // keys is empty + } - while (await ExceptionWrapper.WrapReadAsync(reader, cancellationToken, "GetBatch", this.CollectionName).ConfigureAwait(false)) - { - yield return this._mapper.MapFromStorageToDataModel( - new SqlDataReaderDictionary(reader, this._propertyReader.VectorPropertyStoragePropertyNames), - new() { IncludeVectors = includeVectors }); - } + checked + { + taken += command.Parameters.Count; + } + + using SqlDataReader reader = await ExceptionWrapper.WrapAsync(connection, command, + static (cmd, ct) => cmd.ExecuteReaderAsync(ct), + cancellationToken, "GetBatch", this.CollectionName).ConfigureAwait(false); + + while (await ExceptionWrapper.WrapReadAsync(reader, cancellationToken, "GetBatch", this.CollectionName).ConfigureAwait(false)) + { + yield return this._mapper.MapFromStorageToDataModel( + new SqlDataReaderDictionary(reader, this._propertyReader.VectorPropertyStoragePropertyNames), + new() { IncludeVectors = includeVectors }); + } + } while (command.Parameters.Count == SqlServerConstants.MaxParameterCount); } /// @@ -291,26 +349,84 @@ public async IAsyncEnumerable UpsertBatchAsync(IEnumerable record Verify.NotNull(records); using SqlConnection connection = new(this._connectionString); - using SqlCommand? command = SqlServerCommandBuilder.MergeIntoMany( - connection, - this._options.Schema, - this.CollectionName, - this._propertyReader.KeyProperty, - this._propertyReader.Properties, - records.Select(record => this._mapper.MapFromDataToStorageModel(record))); + await connection.OpenAsync(cancellationToken).ConfigureAwait(false); + + using SqlTransaction transaction = connection.BeginTransaction(); + int parametersPerRecord = this._propertyReader.Properties.Count; + int taken = 0; - if (command is null) + try { - yield break; // records is empty + while (true) + { +#if NET + SqlCommand command = new("", connection, transaction); + await using (command.ConfigureAwait(false)) +#else + using (SqlCommand command = new("", connection, transaction)) +#endif + { + if (!SqlServerCommandBuilder.MergeIntoMany( + command, + this._options.Schema, + this.CollectionName, + this._propertyReader.KeyProperty, + this._propertyReader.Properties, + records.Skip(taken) + .Take(SqlServerConstants.MaxParameterCount / parametersPerRecord) + .Select(this._mapper.MapFromDataToStorageModel))) + { + break; // records is empty + } + + checked + { + taken += (command.Parameters.Count / parametersPerRecord); + } + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + if (taken > 0) + { +#if NET + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); +#else + transaction.Commit(); +#endif + } } + catch (Exception ex) + { +#if NET + await transaction.RollbackAsync(cancellationToken).ConfigureAwait(false); +#else + transaction.Rollback(); +#endif - using SqlDataReader reader = await ExceptionWrapper.WrapAsync(connection, command, - static (cmd, ct) => cmd.ExecuteReaderAsync(ct), - cancellationToken, "GetBatch", this.CollectionName).ConfigureAwait(false); + throw new VectorStoreOperationException(ex.Message, ex) + { + OperationName = "UpsertBatch", + VectorStoreType = ExceptionWrapper.VectorStoreType, + CollectionName = this.CollectionName + }; + } - while (await ExceptionWrapper.WrapReadAsync(reader, cancellationToken, "GetBatch", this.CollectionName).ConfigureAwait(false)) + if (typeof(TRecord) == typeof(VectorStoreGenericDataModel)) + { + foreach (var record in records) + { + yield return ((VectorStoreGenericDataModel)(object)record!).Key; + } + } + else { - yield return reader.GetFieldValue(0); + var keyProperty = this._propertyReader.KeyPropertyInfo; + foreach (var record in records) + { + yield return (TKey)keyProperty.GetValue(record)!; + } } } diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs index 1914a08b74fe..c2d71d49281b 100644 --- a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs @@ -1,7 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. +using Microsoft.Extensions.VectorData; using SqlServerIntegrationTests.Support; using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Models; +using VectorDataSpecificationTests.Xunit; using Xunit; namespace SqlServerIntegrationTests.CRUD; @@ -9,4 +12,63 @@ namespace SqlServerIntegrationTests.CRUD; public class SqlServerBatchConformanceTests(SqlServerSimpleModelFixture fixture) : BatchConformanceTests(fixture), IClassFixture { + private const int SqlServerMaxParameters = 2_100; + + [ConditionalFact] + public Task CanSplitBatchToAccountForMaxParameterLimit_WithVectors() + => this.CanSplitBatchToAccountForMaxParameterLimit(includeVectors: true); + + [ConditionalFact] + public Task CanSplitBatchToAccountForMaxParameterLimit_WithoutVectors() + => this.CanSplitBatchToAccountForMaxParameterLimit(includeVectors: false); + + private async Task CanSplitBatchToAccountForMaxParameterLimit(bool includeVectors) + { + var collection = fixture.Collection; + SimpleModel[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleModel() + { + Id = fixture.GenerateNextKey(), + Number = 100 + i, + Text = i.ToString(), + Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + }).ToArray(); + var keys = inserted.Select(record => record.Id).ToArray(); + + Assert.Empty(await collection.GetBatchAsync(keys).ToArrayAsync()); + var receivedKeys = await collection.UpsertBatchAsync(inserted).ToArrayAsync(); + Assert.Equal(keys.ToHashSet(), receivedKeys.ToHashSet()); // .ToHashSet() to ignore order + + var received = await collection.GetBatchAsync(keys, new() { IncludeVectors = includeVectors }).ToArrayAsync(); + foreach (var record in inserted) + { + record.AssertEqual(this.GetRecord(received, record.Id), includeVectors); + } + + await collection.DeleteBatchAsync(keys); + Assert.Empty(await collection.GetBatchAsync(keys).ToArrayAsync()); + } + + [ConditionalFact] + public async Task UpsertBatchIsAtomic() + { + var collection = fixture.Collection; + SimpleModel[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleModel() + { + // The last Id is set to NULL, so it must not be inserted and the whole batch should fail + Id = i < SqlServerMaxParameters ? fixture.GenerateNextKey() : null!, + Number = 100 + i, + Text = i.ToString(), + Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + }).ToArray(); + + var keys = inserted.Select(record => record.Id).Where(key => key is not null).ToArray(); + Assert.Empty(await collection.GetBatchAsync(keys).ToArrayAsync()); + + VectorStoreOperationException ex = await Assert.ThrowsAsync(() => collection.UpsertBatchAsync(inserted).ToArrayAsync().AsTask()); + Assert.Equal("UpsertBatch", ex.OperationName); + Assert.Equal(collection.CollectionName, ex.CollectionName); + + // Make sure that no records were inserted! + Assert.Empty(await collection.GetBatchAsync(keys).ToArrayAsync()); + } } diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/SqlServerCommandBuilderTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/SqlServerCommandBuilderTests.cs index e22d9f98a201..c8ed9a0cdda1 100644 --- a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/SqlServerCommandBuilderTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/SqlServerCommandBuilderTests.cs @@ -228,8 +228,10 @@ public void MergeIntoMany() ]; using SqlConnection connection = CreateConnection(); - using SqlCommand command = SqlServerCommandBuilder.MergeIntoMany(connection, "schema", "table", - keyProperty, properties, records)!; + using SqlCommand command = connection.CreateCommand(); + + Assert.True(SqlServerCommandBuilder.MergeIntoMany(command, "schema", "table", + keyProperty, properties, records)); string expectedCommand = """" @@ -283,9 +285,9 @@ public void DeleteMany() string[] keys = ["key1", "key2"]; VectorStoreRecordKeyProperty keyProperty = new("id", typeof(string)); using SqlConnection connection = CreateConnection(); + using SqlCommand command = connection.CreateCommand(); - using SqlCommand command = SqlServerCommandBuilder.DeleteMany(connection, - "schema", "tableName", keyProperty, keys)!; + Assert.True(SqlServerCommandBuilder.DeleteMany(command, "schema", "tableName", keyProperty, keys)); Assert.Equal("DELETE FROM [schema].[tableName] WHERE [id] IN (@id_0,@id_1)", command.CommandText); for (int i = 0; i < keys.Length; i++) @@ -338,9 +340,10 @@ public void SelectMany() ]; long[] keys = [123L, 456L, 789L]; using SqlConnection connection = CreateConnection(); + using SqlCommand command = connection.CreateCommand(); - using SqlCommand command = SqlServerCommandBuilder.SelectMany(connection, - "schema", "tableName", keyProperty, properties, keys, includeVectors: true)!; + Assert.True(SqlServerCommandBuilder.SelectMany(command, + "schema", "tableName", keyProperty, properties, keys, includeVectors: true)); AssertEqualIgnoreNewLines( """"" diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs index ee5ded61809e..b8fe0a30afe4 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs @@ -196,6 +196,6 @@ public async Task DeleteBatchAsyncDeletesTheRecords() // The order of records in the received array is not guaranteed // to match the order of keys in the requested keys array. - private SimpleModel GetRecord(SimpleModel[] received, TKey key) + protected SimpleModel GetRecord(SimpleModel[] received, TKey key) => received.Single(r => r.Id!.Equals(key)); } From 2cade4f60c1351190e440d114cab5762ba4806a7 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Wed, 19 Mar 2025 11:28:49 -0700 Subject: [PATCH 11/19] Fixed SQL Server tests --- .../CRUD/SqlServerBatchConformanceTests.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs index c2d71d49281b..68c5d0100410 100644 --- a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs @@ -66,7 +66,11 @@ public async Task UpsertBatchIsAtomic() VectorStoreOperationException ex = await Assert.ThrowsAsync(() => collection.UpsertBatchAsync(inserted).ToArrayAsync().AsTask()); Assert.Equal("UpsertBatch", ex.OperationName); - Assert.Equal(collection.CollectionName, ex.CollectionName); + + var metadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; + + Assert.NotNull(metadata?.CollectionName); + Assert.Equal(metadata.CollectionName, ex.CollectionName); // Make sure that no records were inserted! Assert.Empty(await collection.GetBatchAsync(keys).ToArrayAsync()); From f85c21787f49206e21a28d99527af015b6c9e8ea Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Wed, 19 Mar 2025 14:20:52 -0700 Subject: [PATCH 12/19] .Net: Fixed links (#11068) ### Motivation and Context This PR fixes links that were found unreachable by `linkspector`: https://github.com/microsoft/semantic-kernel/actions/runs/13954045797/job/39063140670?pr=11055 ### Contribution Checklist - [x] The code builds clean without any errors or warnings - [x] The PR follows the [SK Contribution Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md) and the [pre-submission formatting script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts) raises no violations - [x] All unit tests pass, and I have added new tests where possible - [x] I didn't break anyone :smile: --- .github/.linkspector.yml | 12 ++++++++---- README.md | 2 +- docs/PLUGINS.md | 4 ++-- docs/PROMPT_TEMPLATE_LANGUAGE.md | 4 ++-- .../0008-support-generic-llm-request-settings.md | 2 +- dotnet/samples/Concepts/README.md | 14 ++------------ dotnet/samples/Demos/AIModelRouter/README.md | 2 +- dotnet/samples/Demos/BookingRestaurant/README.md | 4 ++-- dotnet/samples/Demos/TimePlugin/README.md | 4 ++-- dotnet/samples/GettingStartedWithAgents/README.md | 10 +++++----- 10 files changed, 26 insertions(+), 32 deletions(-) diff --git a/.github/.linkspector.yml b/.github/.linkspector.yml index 89cf32cfb050..6bde302ae51a 100644 --- a/.github/.linkspector.yml +++ b/.github/.linkspector.yml @@ -7,10 +7,14 @@ ignorePatterns: - pattern: "./issues" - pattern: "./discussions" - pattern: "./pulls" - - pattern: "^http://localhost" - - pattern: "^https://localhost" - - pattern: "^https://platform.openai.com" - - pattern: "^https://outlook.office.com/bookings" + - pattern: "https:\/\/platform.openai.com" + - pattern: "https:\/\/outlook.office.com/bookings" +excludedDirs: + # Folders which include links to localhost, since it's not ignored with regular expressions + - ./python/samples/demos/telemetry + - ./python/samples/demos/process_with_dapr + - ./dotnet/samples/Demos/ProcessWithDapr + - ./dotnet/samples/Demos/CopilotAgentPlugins baseUrl: https://github.com/microsoft/semantic-kernel/ aliveStatusCodes: - 200 diff --git a/README.md b/README.md index 85f771ffb218..ee2694e58b53 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ - Python
[![Python package](https://img.shields.io/pypi/v/semantic-kernel)](https://pypi.org/project/semantic-kernel/) - .NET
- [![Nuget package](https://img.shields.io/nuget/vpre/Microsoft.SemanticKernel)](https://www.nuget.org/packages/Microsoft.SemanticKernel/)[![dotnet Docker](https://github.com/microsoft/semantic-kernel/actions/workflows/dotnet-ci-docker.yml/badge.svg?branch=main)](https://github.com/microsoft/semantic-kernel/actions/workflows/dotnet-ci-docker.yml)[![dotnet Windows](https://github.com/microsoft/semantic-kernel/actions/workflows/dotnet-ci-windows.yml/badge.svg?branch=main)](https://github.com/microsoft/semantic-kernel/actions/workflows/dotnet-ci-windows.yml) + [![Nuget package](https://img.shields.io/nuget/vpre/Microsoft.SemanticKernel)](https://www.nuget.org/packages/Microsoft.SemanticKernel/) ## Overview diff --git a/docs/PLUGINS.md b/docs/PLUGINS.md index 2407b01ce3e4..7926c645c1bd 100644 --- a/docs/PLUGINS.md +++ b/docs/PLUGINS.md @@ -1,5 +1,5 @@ # What are plugins? -This document has been moved to the Semantic Kernel Documentation site. You can find it by navigating to the [Using AI plugins in Semantic Kernel](https://learn.microsoft.com/en-us/semantic-kernel/ai-orchestration/plugins) page. +This document has been moved to the Semantic Kernel Documentation site. You can find it by navigating to the [What is a Plugin?](https://learn.microsoft.com/en-us/semantic-kernel/concepts/plugins) page. -To make an update on the page, file a PR on the [docs repo.](https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/semantic-kernel/ai-orchestration/plugins.md) +To make an update on the page, file a PR on the [docs repo.](https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/semantic-kernel/concepts/plugins/index.md) diff --git a/docs/PROMPT_TEMPLATE_LANGUAGE.md b/docs/PROMPT_TEMPLATE_LANGUAGE.md index 4a5c00784cdc..42201f7e8523 100644 --- a/docs/PROMPT_TEMPLATE_LANGUAGE.md +++ b/docs/PROMPT_TEMPLATE_LANGUAGE.md @@ -1,5 +1,5 @@ # SK Prompt Template Syntax -This document has been moved to the Semantic Kernel Documentation site. You can find it by navigating to the [Prompt template syntax](https://learn.microsoft.com/en-us/semantic-kernel/prompt-engineering/prompt-template-syntax) page. +This document has been moved to the Semantic Kernel Documentation site. You can find it by navigating to the [What are prompts?](https://learn.microsoft.com/en-us/semantic-kernel/concepts/prompts) page. -To make an update on the page, file a PR on the [docs repo.](https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/semantic-kernel/prompt-engineering/prompt-template-syntax.md) +To make an update on the page, file a PR on the [docs repo.](https://github.com/MicrosoftDocs/semantic-kernel-docs/blob/main/semantic-kernel/concepts/prompts/index.md) diff --git a/docs/decisions/0008-support-generic-llm-request-settings.md b/docs/decisions/0008-support-generic-llm-request-settings.md index 6fae2fdf3ef5..f786884f5d9d 100644 --- a/docs/decisions/0008-support-generic-llm-request-settings.md +++ b/docs/decisions/0008-support-generic-llm-request-settings.md @@ -12,7 +12,7 @@ informed: ## Context and Problem Statement -The Semantic Kernel abstractions package includes a number of classes ([CompleteRequestSettings](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/CompleteRequestSettings.cs), [ChatRequestSettings](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatRequestSettings.cs) [PromptTemplateConfig.CompletionConfig](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/SemanticFunctions/PromptTemplateConfig.cs#L18C1-L82C6)) which are used to support: +The Semantic Kernel abstractions package includes a number of classes (`CompleteRequestSettings`, `ChatRequestSettings`, `PromptTemplateConfig.CompletionConfig`) which are used to support: 1. Passing LLM request settings when invoking an AI service 2. Deserialization of LLM requesting settings when loading the `config.json` associated with a Semantic Function diff --git a/dotnet/samples/Concepts/README.md b/dotnet/samples/Concepts/README.md index 2c213d423790..7f6f3b4479b6 100644 --- a/dotnet/samples/Concepts/README.md +++ b/dotnet/samples/Concepts/README.md @@ -21,17 +21,8 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom ### Agents - Different ways of using [`Agents`](./Agents/README.md) - [ComplexChat_NestedShopper](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/ComplexChat_NestedShopper.cs) -- [Legacy_AgentAuthoring](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentAuthoring.cs) -- [Legacy_AgentCharts](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentCharts.cs) -- [Legacy_AgentCollaboration](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentCollaboration.cs) -- [Legacy_AgentDelegation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentDelegation.cs) -- [Legacy_AgentTools](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_AgentTools.cs) -- [Legacy_Agents](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_Agents.cs) -- [Legacy_ChatCompletionAgent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/Legacy_ChatCompletionAgent.cs) - [MixedChat_Agents](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/MixedChat_Agents.cs) - [OpenAIAssistant_ChartMaker](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_ChartMaker.cs) -- [OpenAIAssistant_CodeInterpreter](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_CodeInterpreter.cs) -- [OpenAIAssistant_Retrieval](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Agents/OpenAIAssistant_Retrieval.cs) ### AudioToText - Different ways of using [`AudioToText`](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/AudioToText/IAudioToTextService.cs) services to extract text from audio @@ -78,7 +69,7 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom - [MistralAI_ChatPrompt](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_ChatPrompt.cs) - [MistralAI_FunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_FunctionCalling.cs) - [MistralAI_StreamingFunctionCalling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MistralAI_StreamingFunctionCalling.cs) -- [MultipleProviders_ChatHistoryReducer](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MuiltipleProviders_ChatHistoryReducer.cs) +- [MultipleProviders_ChatHistoryReducer](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/MultipleProviders_ChatHistoryReducer.cs) - [Ollama_ChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Ollama_ChatCompletion.cs) - [Ollama_ChatCompletionStreaming](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Ollama_ChatCompletionStreaming.cs) - [Onnx_ChatCompletion](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/Onnx_ChatCompletion.cs) @@ -106,9 +97,8 @@ dotnet test -l "console;verbosity=detailed" --filter "FullyQualifiedName=ChatCom - [AutoFunctionInvocationFiltering](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/AutoFunctionInvocationFiltering.cs) - [FunctionInvocationFiltering](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/FunctionInvocationFiltering.cs) -- [Legacy_KernelHooks](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/Legacy_KernelHooks.cs) - [MaxTokensWithFilters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/MaxTokensWithFilters.cs) -- [PIIDetectionWithFilters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PIIDetectionWithFilters.cs) +- [PIIDetection](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PIIDetection.cs) - [PromptRenderFiltering](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/PromptRenderFiltering.cs) - [RetryWithFilters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/RetryWithFilters.cs) - [TelemetryWithFilters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/Filtering/TelemetryWithFilters.cs) diff --git a/dotnet/samples/Demos/AIModelRouter/README.md b/dotnet/samples/Demos/AIModelRouter/README.md index 18c556db1e15..6ee34e609bec 100644 --- a/dotnet/samples/Demos/AIModelRouter/README.md +++ b/dotnet/samples/Demos/AIModelRouter/README.md @@ -7,7 +7,7 @@ This sample demonstrates how to implement an AI Model Router using Semantic Kern ## Semantic Kernel Features Used -- [Chat Completion Service](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs) - Using the Chat Completion Service [OpenAI Connector implementation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.OpenAI/ChatCompletion/OpenAIChatCompletionService.cs) to generate responses from the LLM. +- [Chat Completion Service](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs) - Using the Chat Completion Service [OpenAI Connector implementation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIChatCompletionService.cs) to generate responses from the LLM. - [Filters](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs), using to capture selected service and log in the console. ## Prerequisites diff --git a/dotnet/samples/Demos/BookingRestaurant/README.md b/dotnet/samples/Demos/BookingRestaurant/README.md index 34cefed54126..a0d60175b505 100644 --- a/dotnet/samples/Demos/BookingRestaurant/README.md +++ b/dotnet/samples/Demos/BookingRestaurant/README.md @@ -5,9 +5,9 @@ This sample provides a practical demonstration of how to leverage features from ## Semantic Kernel Features Used - [Plugin](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/Functions/KernelPlugin.cs) - Creating a Plugin from a native C# Booking class to be used by the Kernel to interact with Bookings API. -- [Chat Completion Service](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs) - Using the Chat Completion Service [OpenAI Connector implementation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.OpenAI/ChatCompletion/OpenAIChatCompletionService.cs) to generate responses from the LLM. +- [Chat Completion Service](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs) - Using the Chat Completion Service [OpenAI Connector implementation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIChatCompletionService.cs) to generate responses from the LLM. - [Chat History](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatHistory.cs) Using the Chat History abstraction to create, update and retrieve chat history from Chat Completion Models. -- [Auto Function Calling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/AutoFunctionCalling/OpenAI_FunctionCalling.cs) Enables the LLM to have knowledge of current importedUsing the Function Calling feature automatically call the Booking Plugin from the LLM. +- [Auto Function Calling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_FunctionCalling.cs) Enables the LLM to have knowledge of current importedUsing the Function Calling feature automatically call the Booking Plugin from the LLM. ## Prerequisites diff --git a/dotnet/samples/Demos/TimePlugin/README.md b/dotnet/samples/Demos/TimePlugin/README.md index 335d2429aaea..3ed1cd6fb97d 100644 --- a/dotnet/samples/Demos/TimePlugin/README.md +++ b/dotnet/samples/Demos/TimePlugin/README.md @@ -8,9 +8,9 @@ Here we have a simple Time Plugin created in C# that can be called from the AI M ## Semantic Kernel Features Used - [Plugin](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/Functions/KernelPlugin.cs) - Creating a Plugin from a native C# Booking class to be used by the Kernel to interact with Bookings API. -- [Chat Completion Service](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs) - Using the Chat Completion Service [OpenAI Connector implementation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.OpenAI/ChatCompletion/OpenAIChatCompletionService.cs) to generate responses from the LLM. +- [Chat Completion Service](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletionService.cs) - Using the Chat Completion Service [OpenAI Connector implementation](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/Connectors/Connectors.OpenAI/Services/OpenAIChatCompletionService.cs) to generate responses from the LLM. - [Chat History](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatHistory.cs) Using the Chat History abstraction to create, update and retrieve chat history from Chat Completion Models. -- [Auto Function Calling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/AutoFunctionCalling/OpenAI_FunctionCalling.cs) Enables the LLM to have knowledge of current importedUsing the Function Calling feature automatically call the Booking Plugin from the LLM. +- [Auto Function Calling](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/Concepts/ChatCompletion/OpenAI_FunctionCalling.cs) Enables the LLM to have knowledge of current importedUsing the Function Calling feature automatically call the Booking Plugin from the LLM. ## Prerequisites diff --git a/dotnet/samples/GettingStartedWithAgents/README.md b/dotnet/samples/GettingStartedWithAgents/README.md index 6c54a26c0d90..207dfde552be 100644 --- a/dotnet/samples/GettingStartedWithAgents/README.md +++ b/dotnet/samples/GettingStartedWithAgents/README.md @@ -37,7 +37,7 @@ Example|Description [Step01_Assistant](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/OpenAIAssistant/Step01_Assistant.cs)|How to create an Open AI Assistant agent. [Step02_Assistant_Plugins](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/OpenAIAssistant/Step02_Assistant_Plugins.cs)|How to create an Open AI Assistant agent. [Step03_Assistant_Vision](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/OpenAIAssistant/Step03_Assistant_Vision.cs)|How to provide an image as input to an Open AI Assistant agent. -[Step04_AssistantTool_CodeInterpreter_](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/OpenAIAssistant/Step04_AssistantTool_CodeInterpreter_.cs)|How to use the code-interpreter tool for an Open AI Assistant agent. +[Step04_AssistantTool_CodeInterpreter_](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/OpenAIAssistant/Step04_AssistantTool_CodeInterpreter.cs)|How to use the code-interpreter tool for an Open AI Assistant agent. [Step05_AssistantTool_FileSearch](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/OpenAIAssistant/Step05_AssistantTool_FileSearch.cs)|How to use the file-search tool for an Open AI Assistant agent. ### Azure AI Agent @@ -46,10 +46,10 @@ Example|Description ---|--- [Step01_AzureAIAgent](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step01_AzureAIAgent.cs)|How to create an Azure AI agent. [Step02_AzureAIAgent_Plugins](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step02_AzureAIAgent_Plugins.cs)|How to create an Azure AI agent. -[Step03_AzureAIAgent_Chat](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step02_AzureAIAgent_Chat.cs)|How create a conversation with Azure AI agents. -[Step04_AzureAIAgent_CodeInterpreter](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step03_AzureAIAgent_CodeInterpreter.cs)|How to use the code-interpreter tool for an Azure AI agent. -[Step05_AzureAIAgent_FileSearch](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step04_AzureAIAgent_FileSearch.cs)|How to use the file-search tool for an Azure AI agent. -[Step06_AzureAIAgent_OpenAPI](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step05_AzureAIAgent_OpenAPI.cs)|How to use the Open API tool for an Azure AI agent. +[Step03_AzureAIAgent_Chat](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step03_AzureAIAgent_Chat.cs)|How create a conversation with Azure AI agents. +[Step04_AzureAIAgent_CodeInterpreter](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step04_AzureAIAgent_CodeInterpreter.cs)|How to use the code-interpreter tool for an Azure AI agent. +[Step05_AzureAIAgent_FileSearch](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step05_AzureAIAgent_FileSearch.cs)|How to use the file-search tool for an Azure AI agent. +[Step06_AzureAIAgent_OpenAPI](https://github.com/microsoft/semantic-kernel/blob/main/dotnet/samples/GettingStartedWithAgents/AzureAIAgent/Step06_AzureAIAgent_OpenAPI.cs)|How to use the Open API tool for an Azure AI agent. ### Bedrock Agent From dd20b69c287d5b39cf8ac921cfbee4c41c4e68f0 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Wed, 19 Mar 2025 18:06:16 -0700 Subject: [PATCH 13/19] Addressed PR feedback --- .../AzureAISearchVectorStore.cs | 14 ++++++------- ...zureAISearchVectorStoreRecordCollection.cs | 14 ++++++------- .../AzureCosmosDBMongoDBVectorStore.cs | 2 +- ...mosDBMongoDBVectorStoreRecordCollection.cs | 16 +++++++-------- .../AzureCosmosDBNoSQLVectorStore.cs | 2 +- ...osmosDBNoSQLVectorStoreRecordCollection.cs | 10 +++++----- .../InMemoryVectorStore.cs | 4 ++-- .../InMemoryVectorStoreRecordCollection.cs | 2 +- .../MongoDBVectorStore.cs | 2 +- .../MongoDBVectorStoreRecordCollection.cs | 20 +++++++++---------- .../PineconeVectorStore.cs | 4 ++-- .../PineconeVectorStoreRecordCollection.cs | 14 ++++++------- .../PostgresVectorStore.cs | 6 +++--- .../PostgresVectorStoreRecordCollection.cs | 18 ++++++++--------- .../PostgresVectorStoreUtils.cs | 14 ++++++------- .../QdrantVectorStore.cs | 4 ++-- .../QdrantVectorStoreRecordCollection.cs | 20 +++++++++---------- ...RedisHashSetVectorStoreRecordCollection.cs | 14 ++++++------- .../RedisJsonVectorStoreRecordCollection.cs | 18 ++++++++--------- .../RedisVectorStore.cs | 4 ++-- .../SqlServerVectorStore.cs | 2 +- .../SqlServerVectorStoreRecordCollection.cs | 2 +- .../SqliteVectorStore.cs | 2 +- .../SqliteVectorStoreRecordCollection.cs | 10 +++++----- .../WeaviateVectorStore.cs | 2 +- .../WeaviateVectorStoreRecordCollection.cs | 12 +++++------ .../KeywordHybridSearchMetadata.cs | 10 +++++----- .../VectorizableTextSearchMetadata.cs | 10 +++++----- .../VectorSearch/VectorizedSearchMetadata.cs | 10 +++++----- .../VectorStorage/VectorStoreMetadata.cs | 8 ++++---- .../VectorStoreRecordCollectionMetadata.cs | 8 ++++---- .../Collections/CollectionConformanceTests.cs | 6 +++--- 32 files changed, 142 insertions(+), 142 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs index ce0bb6e72386..f863bb8340c3 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStore.cs @@ -44,7 +44,7 @@ public AzureAISearchVectorStore(SearchIndexClient searchIndexClient, AzureAISear this._metadata = new() { - VectorStoreName = "azure.aisearch", + VectorStoreSystemName = "azure.aisearch", DatabaseName = searchIndexClient.ServiceName }; } @@ -83,11 +83,11 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat var indexNamesEnumerable = this._searchIndexClient.GetIndexNamesAsync(cancellationToken).ConfigureAwait(false); var indexNamesEnumerator = indexNamesEnumerable.GetAsyncEnumerator(); - var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreName).ConfigureAwait(false); + var nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreSystemName).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.name; - nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreName).ConfigureAwait(false); + nextResult = await GetNextIndexNameAsync(indexNamesEnumerator, this._metadata.VectorStoreSystemName).ConfigureAwait(false); } } @@ -110,11 +110,11 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat /// around a yield return. /// /// The enumerator to get the next result from. - /// The vector store name. + /// The vector store system name. /// A value indicating whether there are more results and the current string if true. private static async Task<(string name, bool more)> GetNextIndexNameAsync( ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, - string? vectorStoreName) + string? vectorStoreSystemName) { const string OperationName = "GetIndexNames"; @@ -127,7 +127,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = vectorStoreName, + VectorStoreType = vectorStoreSystemName, OperationName = OperationName }; } @@ -135,7 +135,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = vectorStoreName, + VectorStoreType = vectorStoreSystemName, OperationName = OperationName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs index 97d1af5bd6fc..df92bb064e7e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs @@ -155,7 +155,7 @@ public AzureAISearchVectorStoreRecordCollection(SearchIndexClient searchIndexCli this._collectionMetadata = new() { - VectorStoreName = "azure.aisearch", + VectorStoreSystemName = "azure.aisearch", DatabaseName = searchIndexClient.ServiceName, CollectionName = collectionName }; @@ -184,7 +184,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionMetadata.CollectionName, OperationName = "GetIndex" }; @@ -556,7 +556,7 @@ public Task> HybridSearchAsync(TVector vec } return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, OperationName, () => this._mapper!.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); @@ -619,7 +619,7 @@ private Task> MapToStorageModelAndUploadDocumentA if (this._mapper is not null) { var jsonObjects = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, OperationName, () => records.Select(this._mapper!.MapFromDataToStorageModel)); @@ -647,7 +647,7 @@ private async IAsyncEnumerable> MapSearchResultsAsyn await foreach (var result in results.ConfigureAwait(false)) { var document = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, operationName, () => this._options.JsonObjectCustomMapper!.MapFromStorageToDataModel(result.Document, new() { IncludeVectors = includeVectors })); @@ -727,7 +727,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionMetadata.CollectionName, OperationName = operationName }; @@ -736,7 +736,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionMetadata.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs index 4c7efaba0aa4..7d86b598da80 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStore.cs @@ -42,7 +42,7 @@ public AzureCosmosDBMongoDBVectorStore(IMongoDatabase mongoDatabase, AzureCosmos this._metadata = new() { - VectorStoreName = "azure.cosmosdbmongodb", + VectorStoreSystemName = "azure.cosmosdbmongodb", DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs index e060883cc5f0..e32cd08eb588 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs @@ -101,7 +101,7 @@ public AzureCosmosDBMongoDBVectorStoreRecordCollection( this._collectionMetadata = new() { - VectorStoreName = "azure.cosmosdbmongodb", + VectorStoreSystemName = "azure.cosmosdbmongodb", DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, CollectionName = collectionName }; @@ -122,7 +122,7 @@ public virtual async Task CreateCollectionAsync(CancellationToken cancellationTo { throw new VectorStoreOperationException("Collection already exists.") { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionMetadata.CollectionName, OperationName = "CreateCollection" }; @@ -187,7 +187,7 @@ public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = } return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new() { IncludeVectors = includeVectors })); @@ -214,7 +214,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( if (record is not null) { yield return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new())); @@ -232,7 +232,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancel var replaceOptions = new ReplaceOptions { IsUpsert = true }; var storageModel = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -428,7 +428,7 @@ private async IAsyncEnumerable> EnumerateAndMapSearc { var score = response[ScorePropertyName].AsDouble; var record = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, new())); @@ -467,7 +467,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -484,7 +484,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs index 011b69ff95f8..6127b3936ea6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStore.cs @@ -42,7 +42,7 @@ public AzureCosmosDBNoSQLVectorStore(Database database, AzureCosmosDBNoSQLVector this._metadata = new() { - VectorStoreName = "azure.cosmosdbnosql", + VectorStoreSystemName = "azure.cosmosdbnosql", DatabaseName = database.Id }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs index c7c3aac1b3b8..701057f52673 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs @@ -175,7 +175,7 @@ public AzureCosmosDBNoSQLVectorStoreRecordCollection( this._collectionMetadata = new() { - VectorStoreName = "azure.cosmosdbnosql", + VectorStoreSystemName = "azure.cosmosdbnosql", DatabaseName = database.Id, CollectionName = collectionName }; @@ -501,7 +501,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -688,7 +688,7 @@ private async IAsyncEnumerable InternalGetAsync( await foreach (var jsonObject in this.GetItemsAsync(queryDefinition, cancellationToken).ConfigureAwait(false)) { yield return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); @@ -704,7 +704,7 @@ private async Task InternalUpsertAsync( const string OperationName = "UpsertItem"; var jsonObject = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -784,7 +784,7 @@ private async IAsyncEnumerable> MapSearchResultsAsyn jsonObject.Remove(scorePropertyName); var record = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(jsonObject, new() { IncludeVectors = includeVectors })); diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs index 86b676d363df..c974b6aa5809 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStore.cs @@ -34,7 +34,7 @@ public InMemoryVectorStore() this._metadata = new() { - VectorStoreName = "inmemory" + VectorStoreSystemName = "inmemory" }; } @@ -48,7 +48,7 @@ internal InMemoryVectorStore(ConcurrentDictionary this._mapper.MapFromStorageToDataModel(record, new() { IncludeVectors = includeVectors })); @@ -226,7 +226,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( if (record is not null) { yield return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(record, new())); @@ -244,7 +244,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancel var replaceOptions = new ReplaceOptions { IsUpsert = true }; var storageModel = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -524,7 +524,7 @@ private async IAsyncEnumerable> EnumerateAndMapSearc { var score = response[ScorePropertyName].AsDouble; var record = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(response[DocumentPropertyName].AsBsonDocument, new() { IncludeVectors = includeVectors })); @@ -563,7 +563,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -580,7 +580,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -611,7 +611,7 @@ private async Task RunOperationWithRetryAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -645,7 +645,7 @@ private async Task RunOperationWithRetryAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs index 4668531ea5cb..fc088a73e3e6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs @@ -43,7 +43,7 @@ public PineconeVectorStore(Sdk.PineconeClient pineconeClient, PineconeVectorStor this._metadata = new() { - VectorStoreName = "pinecone" + VectorStoreSystemName = "pinecone" }; } @@ -82,7 +82,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._metadata.VectorStoreName, + VectorStoreType = this._metadata.VectorStoreSystemName, OperationName = ListCollectionsName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 63dc74146125..8fdbacc466f9 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -97,7 +97,7 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st this._collectionMetadata = new() { - VectorStoreName = "pinecone", + VectorStoreSystemName = "pinecone", CollectionName = collectionName }; @@ -180,7 +180,7 @@ public virtual async IAsyncEnumerable GetBatchAsync( () => index.Fetch(keys, indexNamespace, cancellationToken)).ConfigureAwait(false); var records = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, GetOperationName, () => results.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); @@ -223,7 +223,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); var vector = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, UpsertOperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -245,7 +245,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -303,7 +303,7 @@ public virtual async Task> VectorizedSearchAsync @@ -352,7 +352,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -369,7 +369,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs index c011d17480e0..2f067ca132a5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStore.cs @@ -35,7 +35,7 @@ public PostgresVectorStore(NpgsqlDataSource dataSource, PostgresVectorStoreOptio this._metadata = new() { - VectorStoreName = "postgresql", + VectorStoreSystemName = "postgresql", DatabaseName = this._postgresClient.DatabaseName }; } @@ -52,7 +52,7 @@ internal PostgresVectorStore(IPostgresVectorStoreDbClient postgresDbClient, Post this._metadata = new() { - VectorStoreName = "postgresql", + VectorStoreSystemName = "postgresql", DatabaseName = this._postgresClient.DatabaseName }; } @@ -64,7 +64,7 @@ public virtual IAsyncEnumerable ListCollectionNamesAsync(CancellationTok return PostgresVectorStoreUtils.WrapAsyncEnumerableAsync( this._postgresClient.GetTablesAsync(cancellationToken), OperationName, - vectorStoreName: this._metadata.VectorStoreName + vectorStoreSystemName: this._metadata.VectorStoreSystemName ); } diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs index de7f495942ab..62afddf824df 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordCollection.cs @@ -113,7 +113,7 @@ internal PostgresVectorStoreRecordCollection(IPostgresVectorStoreDbClient client this._collectionMetadata = new() { - VectorStoreName = "postgresql", + VectorStoreSystemName = "postgresql", DatabaseName = this._client.DatabaseName, CollectionName = collectionName }; @@ -163,7 +163,7 @@ public virtual Task UpsertAsync(TRecord record, CancellationToken cancella const string OperationName = "Upsert"; var storageModel = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -190,7 +190,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -224,7 +224,7 @@ await this.RunOperationAsync(OperationName, () => if (row is null) { return default; } return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(row, new() { IncludeVectors = includeVectors })); @@ -244,7 +244,7 @@ public virtual IAsyncEnumerable GetBatchAsync(IEnumerable keys, G this._client.GetBatchAsync(this.CollectionName, keys, this._propertyReader.RecordDefinition.Properties, includeVectors, cancellationToken) .SelectAsync(row => VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(row, new() { IncludeVectors = includeVectors })), @@ -252,7 +252,7 @@ public virtual IAsyncEnumerable GetBatchAsync(IEnumerable keys, G ), OperationName, this.CollectionName, - this._collectionMetadata.VectorStoreName + this._collectionMetadata.VectorStoreSystemName ); } @@ -321,7 +321,7 @@ public virtual Task> VectorizedSearchAsync .SelectAsync(result => { var record = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel( @@ -364,7 +364,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -381,7 +381,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs index eddcef924d42..ef4e78cf9237 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreUtils.cs @@ -18,21 +18,21 @@ internal static class PostgresVectorStoreUtils /// The async enumerable to wrap. /// The name of the operation being performed. /// The name of the collection being operated on. - /// The name of the vector store being operated on. + /// The name of the vector store system being operated on. /// An async enumerable that will throw a if an exception is thrown while iterating over the original enumerator. public static async IAsyncEnumerable WrapAsyncEnumerableAsync( IAsyncEnumerable asyncEnumerable, string operationName, string? collectionName = null, - string? vectorStoreName = null) + string? vectorStoreSystemName = null) { var enumerator = asyncEnumerable.ConfigureAwait(false).GetAsyncEnumerator(); - var nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreName).ConfigureAwait(false); + var nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreSystemName).ConfigureAwait(false); while (nextResult.more) { yield return nextResult.item; - nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreName).ConfigureAwait(false); + nextResult = await GetNextAsync(enumerator, operationName, collectionName, vectorStoreSystemName).ConfigureAwait(false); } } @@ -43,13 +43,13 @@ public static async IAsyncEnumerable WrapAsyncEnumerableAsync( /// The enumerator to get the next result from. /// The name of the operation being performed. /// The name of the collection being operated on. - /// The name of the vector store being operated on. + /// The name of the vector store system being operated on. /// A value indicating whether there are more results and the current string if true. public static async Task<(T item, bool more)> GetNextAsync( ConfiguredCancelableAsyncEnumerable.Enumerator enumerator, string operationName, string? collectionName = null, - string? vectorStoreName = null) + string? vectorStoreSystemName = null) { try { @@ -60,7 +60,7 @@ public static async IAsyncEnumerable WrapAsyncEnumerableAsync( { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = vectorStoreName, + VectorStoreType = vectorStoreSystemName, CollectionName = collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs index cf95de6e0f58..051956ba964f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStore.cs @@ -53,7 +53,7 @@ internal QdrantVectorStore(MockableQdrantClient qdrantClient, QdrantVectorStoreO this._metadata = new() { - VectorStoreName = "qdrant" + VectorStoreSystemName = "qdrant" }; } @@ -95,7 +95,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._metadata.VectorStoreName, + VectorStoreType = this._metadata.VectorStoreSystemName, OperationName = "ListCollections" }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs index b09c3f3f0f50..f2f7bd2d3cde 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs @@ -139,7 +139,7 @@ internal QdrantVectorStoreRecordCollection(MockableQdrantClient qdrantClient, st this._collectionMetadata = new() { - VectorStoreName = "qdrant", + VectorStoreSystemName = "qdrant", CollectionName = collectionName }; @@ -369,7 +369,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken c // Create point from record. var pointStruct = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, UpsertName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -388,7 +388,7 @@ async Task IVectorStoreRecordCollection.UpsertAsync(TRecord // Create point from record. var pointStruct = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, UpsertName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -407,7 +407,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -430,7 +430,7 @@ async IAsyncEnumerable IVectorStoreRecordCollection.UpsertB // Create points from records. var pointStructs = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, UpsertName, () => records.Select(this._mapper.MapFromDataToStorageModel).ToList()); @@ -488,7 +488,7 @@ private async IAsyncEnumerable GetBatchByPointIdAsync( } yield return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(pointStruct, new() { IncludeVectors = includeVectors })); @@ -549,7 +549,7 @@ public virtual async Task> VectorizedSearchAsync> HybridSearchAsync(TVect point, this._mapper, internalOptions.IncludeVectors, - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "Query")); @@ -679,7 +679,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; @@ -703,7 +703,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs index a1e0f879780b..64d4670356a8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs @@ -149,7 +149,7 @@ public RedisHashSetVectorStoreRecordCollection(IDatabase database, string collec this._collectionMetadata = new() { - VectorStoreName = "redis", + VectorStoreSystemName = "redis", DatabaseName = database.Database.ToString(), CollectionName = collectionName }; @@ -176,7 +176,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = "FT.INFO" }; @@ -265,7 +265,7 @@ await this.RunOperationAsync("FT.DROPINDEX", // Convert to the caller's data model. return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, operationName, () => @@ -323,7 +323,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken // Map. var redisHashSetRecord = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "HSET", () => this._mapper.MapFromDataToStorageModel(record)); @@ -391,7 +391,7 @@ public virtual async Task> VectorizedSearchAsync @@ -473,7 +473,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; @@ -496,7 +496,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs index 91dd5a9adba7..5c0870494fde 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs @@ -131,7 +131,7 @@ public RedisJsonVectorStoreRecordCollection(IDatabase database, string collectio this._collectionMetadata = new() { - VectorStoreName = "redis", + VectorStoreSystemName = "redis", DatabaseName = database.Database.ToString(), CollectionName = collectionName }; @@ -158,7 +158,7 @@ public virtual async Task CollectionExistsAsync(CancellationToken cancella { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = "FT.INFO" }; @@ -247,7 +247,7 @@ await this.RunOperationAsync("FT.DROPINDEX", // Convert to the caller's data model. return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "GET", () => @@ -296,7 +296,7 @@ public virtual async IAsyncEnumerable GetBatchAsync(IEnumerable // Convert to the caller's data model. yield return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "MGET", () => @@ -340,7 +340,7 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken // Map. var redisJsonRecord = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this._collectionName, "SET", () => @@ -374,7 +374,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable @@ -430,7 +430,7 @@ public virtual async Task> VectorizedSearchAsync @@ -514,7 +514,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; @@ -538,7 +538,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this._collectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs index 17137f87e05d..2366b889b647 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisVectorStore.cs @@ -43,7 +43,7 @@ public RedisVectorStore(IDatabase database, RedisVectorStoreOptions? options = d this._metadata = new() { - VectorStoreName = "redis", + VectorStoreSystemName = "redis", DatabaseName = database.Database.ToString() }; } @@ -90,7 +90,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._metadata.VectorStoreName, + VectorStoreType = this._metadata.VectorStoreSystemName, OperationName = OperationName }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs index 772a28626b6e..4c686b8305fb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStore.cs @@ -42,7 +42,7 @@ public SqlServerVectorStore(string connectionString, SqlServerVectorStoreOptions this._metadata = new() { - VectorStoreName = "microsoft.sql_server", + VectorStoreSystemName = "microsoft.sql_server", DatabaseName = connectionStringBuilder.InitialCatalog }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs index 572138efe209..a233ba45dbc6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.SqlServer/SqlServerVectorStoreRecordCollection.cs @@ -101,7 +101,7 @@ public SqlServerVectorStoreRecordCollection( this._collectionMetadata = new() { - VectorStoreName = "microsoft.sql_server", + VectorStoreSystemName = "microsoft.sql_server", DatabaseName = connectionStringBuilder.InitialCatalog, CollectionName = name }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs index 909b0a833bd0..8a24573538d5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStore.cs @@ -45,7 +45,7 @@ public SqliteVectorStore( this._metadata = new() { - VectorStoreName = "sqlite", + VectorStoreSystemName = "sqlite", DatabaseName = connection.Database }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs index 85a05a62f8b1..e98cc1092fa8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs @@ -120,7 +120,7 @@ public SqliteVectorStoreRecordCollection( this._collectionMetadata = new() { - VectorStoreName = "sqlite", + VectorStoreSystemName = "sqlite", DatabaseName = connection.Database, CollectionName = collectionName }; @@ -529,7 +529,7 @@ private async Task InternalUpsertAsync(TRecord record, CancellationT const string OperationName = "Upsert"; var storageModel = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record)); @@ -552,7 +552,7 @@ private IAsyncEnumerable InternalUpsertBatchAsync(IEnumerable VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -679,7 +679,7 @@ private TRecord GetAndMapRecord( } return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors })); @@ -695,7 +695,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs index 34c4ddfe3b80..dfb03bde493f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -48,7 +48,7 @@ public WeaviateVectorStore(HttpClient httpClient, WeaviateVectorStoreOptions? op this._metadata = new() { - VectorStoreName = "weaviate" + VectorStoreSystemName = "weaviate" }; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index b6a26e7db5c9..6e505c58b624 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -165,7 +165,7 @@ public WeaviateVectorStoreRecordCollection( this._collectionMetadata = new() { - VectorStoreName = "weaviate", + VectorStoreSystemName = "weaviate", CollectionName = collectionName }; @@ -287,7 +287,7 @@ public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken c } return VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromStorageToDataModel(jsonObject!, new() { IncludeVectors = includeVectors })); @@ -329,7 +329,7 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable { var jsonObjects = records.Select(record => VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, OperationName, () => this._mapper.MapFromDataToStorageModel(record))).ToList(); @@ -441,7 +441,7 @@ private async Task> ExecuteQueryAsync(string query, { throw new VectorStoreOperationException($"Error occurred during vector search. Response: {content}") { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; @@ -452,7 +452,7 @@ private async Task> ExecuteQueryAsync(string query, var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!, scorePropertyName); var record = VectorStoreErrorHandler.RunModelConversion( - this._collectionMetadata.VectorStoreName!, + this._collectionMetadata.VectorStoreSystemName!, this.CollectionName, operationName, () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors })); @@ -515,7 +515,7 @@ private async Task RunOperationAsync(string operationName, Func> o { throw new VectorStoreOperationException("Call to vector store failed.", ex) { - VectorStoreType = this._collectionMetadata.VectorStoreName, + VectorStoreType = this._collectionMetadata.VectorStoreSystemName, CollectionName = this.CollectionName, OperationName = operationName }; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs index 42a5b8fad242..ced1fe45cc1e 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs @@ -10,11 +10,11 @@ public class KeywordHybridSearchMetadata { /// The name of the vector store. /// - /// Where possible, this maps to the appropriate name defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems. - /// . + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. /// - public string? VectorStoreName { get; init; } + public string? VectorStoreSystemName { get; init; } /// /// The name of the database. @@ -34,7 +34,7 @@ public static KeywordHybridSearchMetadata From(VectorStoreRecordCollectionMetada { return new() { - VectorStoreName = collectionMetadata.VectorStoreName, + VectorStoreSystemName = collectionMetadata.VectorStoreSystemName, DatabaseName = collectionMetadata.DatabaseName, CollectionName = collectionMetadata.CollectionName }; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs index 041baaf89f83..cdfaae18e5f2 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs @@ -10,11 +10,11 @@ public class VectorizableTextSearchMetadata { /// The name of the vector store. /// - /// Where possible, this maps to the appropriate name defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems. - /// . + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. /// - public string? VectorStoreName { get; init; } + public string? VectorStoreSystemName { get; init; } /// /// The name of the database. @@ -34,7 +34,7 @@ public static VectorizableTextSearchMetadata From(VectorStoreRecordCollectionMet { return new() { - VectorStoreName = collectionMetadata.VectorStoreName, + VectorStoreSystemName = collectionMetadata.VectorStoreSystemName, DatabaseName = collectionMetadata.DatabaseName, CollectionName = collectionMetadata.CollectionName }; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs index 110234eab081..c6e7781c2925 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs @@ -10,11 +10,11 @@ public class VectorizedSearchMetadata { /// The name of the vector store. /// - /// Where possible, this maps to the appropriate name defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems. - /// . + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. /// - public string? VectorStoreName { get; init; } + public string? VectorStoreSystemName { get; init; } /// /// The name of the database. @@ -34,7 +34,7 @@ public static VectorizedSearchMetadata From(VectorStoreRecordCollectionMetadata { return new() { - VectorStoreName = collectionMetadata.VectorStoreName, + VectorStoreSystemName = collectionMetadata.VectorStoreSystemName, DatabaseName = collectionMetadata.DatabaseName, CollectionName = collectionMetadata.CollectionName }; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs index dd7b6b8d49a0..eb7b89df5094 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreMetadata.cs @@ -10,11 +10,11 @@ public class VectorStoreMetadata { /// The name of the vector store. /// - /// Where possible, this maps to the appropriate name defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems. - /// . + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. /// - public string? VectorStoreName { get; init; } + public string? VectorStoreSystemName { get; init; } /// /// The name of the database. diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs index 15dee1549750..f4b3a1aacfed 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorStorage/VectorStoreRecordCollectionMetadata.cs @@ -10,11 +10,11 @@ public class VectorStoreRecordCollectionMetadata { /// The name of the vector store. /// - /// Where possible, this maps to the appropriate name defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems. - /// . + /// Where possible, this maps to the "db.system.name" attribute defined in the + /// OpenTelemetry Semantic Conventions for database calls and systems, see . + /// Example: redis, sqlite, mysql. /// - public string? VectorStoreName { get; init; } + public string? VectorStoreSystemName { get; init; } /// /// The name of the database. diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 4efe7a7a80b8..34be5be73c97 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -92,7 +92,7 @@ private async Task CreateCollection() #pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.VectorStoreSystemName); Assert.NotNull(collectionMetadata.DatabaseName); Assert.NotNull(collectionMetadata.CollectionName); @@ -119,7 +119,7 @@ private async Task CreateCollectionIfNotExistsMoreThanOnce() #pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.VectorStoreSystemName); Assert.NotNull(collectionMetadata.DatabaseName); Assert.NotNull(collectionMetadata.CollectionName); @@ -148,7 +148,7 @@ private async Task CreateCollectionMoreThanOnce() #pragma warning restore SKEXP0020 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreName); + Assert.NotNull(collectionMetadata.VectorStoreSystemName); Assert.NotNull(collectionMetadata.DatabaseName); Assert.NotNull(collectionMetadata.CollectionName); From 8ea1c8489ccb69b06b8e3cb6de457cecf3c54376 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 20 Mar 2025 10:47:11 -0700 Subject: [PATCH 14/19] More updates after merge --- .../VectorSearch/LoggingKeywordHybridSearch.cs | 11 +++++++++++ .../VectorSearch/LoggingVectorizableTextSearch.cs | 11 +++++++++++ .../VectorSearch/LoggingVectorizedSearch.cs | 11 +++++++++++ .../VectorData/VectorStorage/LoggingVectorStore.cs | 11 +++++++++++ .../LoggingVectorStoreRecordCollection.cs | 12 ++++++++++++ 5 files changed, 56 insertions(+) diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs index c05ffde310cc..d5c3d28acb39 100644 --- a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs +++ b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingKeywordHybridSearch.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -44,4 +45,14 @@ public Task> HybridSearchAsync(TVector vec nameof(HybridSearchAsync), () => this._innerSearch.HybridSearchAsync(vector, keywords, options, cancellationToken)); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs index 0dc81080e496..a60b6dd12388 100644 --- a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs +++ b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizableTextSearch.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -43,4 +44,14 @@ public Task> VectorizableTextSearchAsync(string sea nameof(VectorizableTextSearchAsync), () => this._innerSearch.VectorizableTextSearchAsync(searchText, options, cancellationToken)); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs index f0198534d421..3e6c099b2282 100644 --- a/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs +++ b/dotnet/src/Connectors/VectorData/VectorSearch/LoggingVectorizedSearch.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Diagnostics.CodeAnalysis; using System.Threading; using System.Threading.Tasks; @@ -43,4 +44,14 @@ public Task> VectorizedSearchAsync(TVector nameof(VectorizedSearchAsync), () => this._innerSearch.VectorizedSearchAsync(vector, options, cancellationToken)); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs index ea2bca4ca106..7fd38761f352 100644 --- a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs +++ b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStore.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -50,4 +51,14 @@ public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cance () => this._innerStore.ListCollectionNamesAsync(cancellationToken), cancellationToken); } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerStore.GetService(serviceType, serviceKey); + } } diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs index 3d6919280861..36e31d1cfd0d 100644 --- a/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/VectorData/VectorStorage/LoggingVectorStoreRecordCollection.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Threading; @@ -39,6 +40,7 @@ public LoggingVectorStoreRecordCollection(IVectorStoreRecordCollection + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] public string CollectionName => this._innerCollection.CollectionName; /// @@ -114,6 +116,16 @@ public IAsyncEnumerable GetBatchAsync(IEnumerable keys, GetRecord cancellationToken); } + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerCollection.GetService(serviceType, serviceKey); + } + /// public Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) { From df076b1ddcf65d66d9248e14709c1ea486fe41ff Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 20 Mar 2025 10:53:11 -0700 Subject: [PATCH 15/19] Small fix --- .../LoggingVectorStoreRecordCollectionTests.cs | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs index 294ee64555a0..7bc32d9a3dfc 100644 --- a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/LoggingVectorStoreRecordCollectionTests.cs @@ -32,23 +32,6 @@ public void ConstructorThrowsOnNullLogger() Assert.Throws(() => new LoggingVectorStoreRecordCollection(innerCollection, null!)); } - [Fact] - public void CollectionNameReturnsInnerCollectionName() - { - // Arrange - var innerCollection = new Mock>(); - innerCollection.Setup(c => c.CollectionName).Returns("test"); - var logger = new Mock().Object; - var decorator = new LoggingVectorStoreRecordCollection(innerCollection.Object, logger); - - // Act - var name = decorator.CollectionName; - - // Assert - Assert.Equal("test", name); - innerCollection.Verify(c => c.CollectionName, Times.Once()); - } - [Fact] public async Task CollectionExistsDelegatesToInnerCollectionAsync() { From fd907a4d554e005f1c9645b209cbd7f29193126a Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 20 Mar 2025 12:20:01 -0700 Subject: [PATCH 16/19] Removed some of the metadata classes --- ...zureAISearchVectorStoreRecordCollection.cs | 16 ------- ...mosDBMongoDBVectorStoreRecordCollection.cs | 6 --- ...osmosDBNoSQLVectorStoreRecordCollection.cs | 11 ----- .../InMemoryVectorStoreRecordCollection.cs | 6 --- .../MongoDBVectorStoreRecordCollection.cs | 11 ----- .../PineconeVectorStoreRecordCollection.cs | 6 --- .../PostgresVectorStoreRecordCollection.cs | 6 --- .../QdrantVectorStoreRecordCollection.cs | 11 ----- ...RedisHashSetVectorStoreRecordCollection.cs | 6 --- .../RedisJsonVectorStoreRecordCollection.cs | 6 --- .../SqlServerVectorStoreRecordCollection.cs | 6 --- .../SqliteVectorStoreRecordCollection.cs | 6 --- .../WeaviateVectorStoreRecordCollection.cs | 11 ----- .../VectorSearch/IKeywordHybridSearch.cs | 2 +- .../VectorSearch/IVectorizableTextSearch.cs | 2 +- .../VectorSearch/IVectorizedSearch.cs | 2 +- .../KeywordHybridSearchMetadata.cs | 42 ------------------- .../VectorizableTextSearchMetadata.cs | 42 ------------------- .../VectorSearch/VectorizedSearchMetadata.cs | 42 ------------------- 19 files changed, 3 insertions(+), 237 deletions(-) delete mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs delete mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs delete mode 100644 dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs index df92bb064e7e..e7dd228b2446 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureAISearch/AzureAISearchVectorStoreRecordCollection.cs @@ -34,15 +34,6 @@ public class AzureAISearchVectorStoreRecordCollection : /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - - /// Metadata about vectorizable text search. - private readonly VectorizableTextSearchMetadata _vectorizableTextSearchMetadata; - - /// Metadata about keyword hybrid search. - private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; - /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -159,10 +150,6 @@ public AzureAISearchVectorStoreRecordCollection(SearchIndexClient searchIndexCli DatabaseName = searchIndexClient.ServiceName, CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); - this._vectorizableTextSearchMetadata = VectorizableTextSearchMetadata.From(this._collectionMetadata); - this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -518,9 +505,6 @@ public Task> HybridSearchAsync(TVector vec return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : - serviceType == typeof(VectorizableTextSearchMetadata) ? this._vectorizableTextSearchMetadata : - serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : serviceType == typeof(SearchIndexClient) ? this._searchIndexClient : serviceType == typeof(SearchClient) ? this._searchClient : serviceType.IsInstanceOfType(this) ? this : diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs index e32cd08eb588..5ec62d911a1e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBMongoDB/AzureCosmosDBMongoDBVectorStoreRecordCollection.cs @@ -30,9 +30,6 @@ public class AzureCosmosDBMongoDBVectorStoreRecordCollection : IVectorS /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - /// Property name to be used for search similarity score value. private const string ScorePropertyName = "similarityScore"; @@ -105,8 +102,6 @@ public AzureCosmosDBMongoDBVectorStoreRecordCollection( DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -346,7 +341,6 @@ public virtual async Task> VectorizedSearchAsync) ? this._mongoCollection : serviceType.IsInstanceOfType(this) ? this : diff --git a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs index 701057f52673..2bb3775852f5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.AzureCosmosDBNoSQL/AzureCosmosDBNoSQLVectorStoreRecordCollection.cs @@ -33,12 +33,6 @@ public class AzureCosmosDBNoSQLVectorStoreRecordCollection : /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - - /// Metadata about keyword hybrid search. - private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; - /// A of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -179,9 +173,6 @@ public AzureCosmosDBNoSQLVectorStoreRecordCollection( DatabaseName = database.Id, CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); - this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -468,8 +459,6 @@ public Task> HybridSearchAsync(TVector vec return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : - serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : serviceType == typeof(Database) ? this._database : serviceType.IsInstanceOfType(this) ? this : null; diff --git a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs index 773ae791b13f..1effc75a45a0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.InMemory/InMemoryVectorStoreRecordCollection.cs @@ -27,9 +27,6 @@ public sealed class InMemoryVectorStoreRecordCollection : IVector /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - /// A set of types that vectors on the provided model may have. private static readonly HashSet s_supportedVectorTypes = [ @@ -95,8 +92,6 @@ public InMemoryVectorStoreRecordCollection(string collectionName, InMemoryVector VectorStoreSystemName = "inmemory", CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -302,7 +297,6 @@ public async Task> VectorizedSearchAsync(T return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : serviceType == typeof(ConcurrentDictionary>) ? this._internalCollections : serviceType.IsInstanceOfType(this) ? this : null; diff --git a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs index 3375a26495b1..b5c7c6822abb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.MongoDB/MongoDBVectorStoreRecordCollection.cs @@ -29,12 +29,6 @@ public class MongoDBVectorStoreRecordCollection : IVectorStoreRecordCol /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - - /// Metadata about keyword hybrid search. - private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; - /// Property name to be used for search similarity score value. private const string ScorePropertyName = "similarityScore"; @@ -110,9 +104,6 @@ public MongoDBVectorStoreRecordCollection( DatabaseName = mongoDatabase.DatabaseNamespace?.DatabaseName, CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); - this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -399,8 +390,6 @@ public async Task> HybridSearchAsync(TVect return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : - serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : serviceType == typeof(IMongoDatabase) ? this._mongoDatabase : serviceType == typeof(IMongoCollection) ? this._mongoCollection : serviceType.IsInstanceOfType(this) ? this : diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index feefc678f839..92816386cdfc 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -28,9 +28,6 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreRecordCollectionOptions _options; private readonly VectorStoreRecordPropertyReader _propertyReader; @@ -90,8 +87,6 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st VectorStoreSystemName = "pinecone", CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -409,7 +404,6 @@ public virtual async Task> VectorizedSearchAsync : IVectorStoreRe /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - /// Postgres client that is used to interact with the database. private readonly IPostgresVectorStoreDbClient _client; @@ -117,8 +114,6 @@ internal PostgresVectorStoreRecordCollection(IPostgresVectorStoreDbClient client DatabaseName = this._client.DatabaseName, CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -343,7 +338,6 @@ public virtual Task> VectorizedSearchAsync return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : serviceType == typeof(NpgsqlDataSource) ? this._client.DataSource : serviceType.IsInstanceOfType(this) ? this : null; diff --git a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs index f2f7bd2d3cde..3217e6fa16cf 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs @@ -30,12 +30,6 @@ public class QdrantVectorStoreRecordCollection : /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - - /// Metadata about keyword hybrid search. - private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; - /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -142,9 +136,6 @@ internal QdrantVectorStoreRecordCollection(MockableQdrantClient qdrantClient, st VectorStoreSystemName = "qdrant", CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); - this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -656,8 +647,6 @@ public async Task> HybridSearchAsync(TVect return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : - serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : serviceType == typeof(QdrantClient) ? this._qdrantClient.QdrantClient : serviceType.IsInstanceOfType(this) ? this : null; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs index 64d4670356a8..a682c84d615e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisHashSetVectorStoreRecordCollection.cs @@ -27,9 +27,6 @@ public class RedisHashSetVectorStoreRecordCollection : IVectorStoreReco /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -153,8 +150,6 @@ public RedisHashSetVectorStoreRecordCollection(IDatabase database, string collec DatabaseName = database.Database.ToString(), CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -418,7 +413,6 @@ public virtual async Task> VectorizedSearchAsync : IVectorStoreRecordC /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -135,8 +132,6 @@ public RedisJsonVectorStoreRecordCollection(IDatabase database, string collectio DatabaseName = database.Database.ToString(), CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -460,7 +455,6 @@ public virtual async Task> VectorizedSearchAsync /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); private static readonly SqlServerVectorStoreRecordCollectionOptions s_defaultOptions = new(); @@ -105,8 +102,6 @@ public SqlServerVectorStoreRecordCollection( DatabaseName = connectionStringBuilder.InitialCatalog, CollectionName = name }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -501,7 +496,6 @@ public async Task> VectorizedSearchAsync(T return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : serviceType.IsInstanceOfType(this) ? this : null; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs index cd2bf532745e..53eec93755b1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Sqlite/SqliteVectorStoreRecordCollection.cs @@ -27,9 +27,6 @@ public class SqliteVectorStoreRecordCollection : /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - /// The connection string for the SQLite database represented by this . private readonly string _connectionString; @@ -126,8 +123,6 @@ public SqliteVectorStoreRecordCollection( DatabaseName = connectionStringBuilder.DataSource, CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); } /// @@ -362,7 +357,6 @@ public async Task DeleteBatchAsync(IEnumerable keys, CancellationToken c return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : serviceType.IsInstanceOfType(this) ? this : null; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 6e505c58b624..6e488a3c56a7 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -29,12 +29,6 @@ public class WeaviateVectorStoreRecordCollection : IVectorStoreRecordCo /// Metadata about vector store record collection. private readonly VectorStoreRecordCollectionMetadata _collectionMetadata; - /// Metadata about vectorized search. - private readonly VectorizedSearchMetadata _vectorizedSearchMetadata; - - /// Metadata about keyword hybrid search. - private readonly KeywordHybridSearchMetadata _keywordHybridSearchMetadata; - /// A set of types that a key on the provided model may have. private static readonly HashSet s_supportedKeyTypes = [ @@ -168,9 +162,6 @@ public WeaviateVectorStoreRecordCollection( VectorStoreSystemName = "weaviate", CollectionName = collectionName }; - - this._vectorizedSearchMetadata = VectorizedSearchMetadata.From(this._collectionMetadata); - this._keywordHybridSearchMetadata = KeywordHybridSearchMetadata.From(this._collectionMetadata); } /// @@ -420,8 +411,6 @@ public async Task> HybridSearchAsync(TVect return serviceKey is not null ? null : serviceType == typeof(VectorStoreRecordCollectionMetadata) ? this._collectionMetadata : - serviceType == typeof(VectorizedSearchMetadata) ? this._vectorizedSearchMetadata : - serviceType == typeof(KeywordHybridSearchMetadata) ? this._keywordHybridSearchMetadata : serviceType == typeof(HttpClient) ? this._httpClient : serviceType.IsInstanceOfType(this) ? this : null; diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs index 313abfecfeed..265366bf1377 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IKeywordHybridSearch.cs @@ -36,7 +36,7 @@ Task> HybridSearchAsync( /// is . /// /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// including itself or any services it might be wrapping. For example, to access the for the instance, /// may be used to request it. /// [Experimental("SKEXP0020")] diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs index c1bb85b5090b..f29d31ee49cd 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizableTextSearch.cs @@ -32,7 +32,7 @@ Task> VectorizableTextSearchAsync( /// is . /// /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// including itself or any services it might be wrapping. For example, to access the for the instance, /// may be used to request it. /// [Experimental("SKEXP0020")] diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs index 6b77dbb84afb..7a8149e6f69a 100644 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs +++ b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/IVectorizedSearch.cs @@ -33,7 +33,7 @@ Task> VectorizedSearchAsync( /// is . /// /// The purpose of this method is to allow for the retrieval of strongly-typed services that might be provided by the , - /// including itself or any services it might be wrapping. For example, to access the for the instance, + /// including itself or any services it might be wrapping. For example, to access the for the instance, /// may be used to request it. /// [Experimental("SKEXP0020")] diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs deleted file mode 100644 index ced1fe45cc1e..000000000000 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/KeywordHybridSearchMetadata.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -[Experimental("SKEXP0020")] -public class KeywordHybridSearchMetadata -{ - /// The name of the vector store. - /// - /// Where possible, this maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems, see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// The name of the database. - /// - public string? DatabaseName { get; init; } - - /// - /// The name of a collection (table, container) within the database. - /// - public string? CollectionName { get; init; } - - /// - /// Initializes an instance of from . - /// - /// Instance of . - public static KeywordHybridSearchMetadata From(VectorStoreRecordCollectionMetadata collectionMetadata) - { - return new() - { - VectorStoreSystemName = collectionMetadata.VectorStoreSystemName, - DatabaseName = collectionMetadata.DatabaseName, - CollectionName = collectionMetadata.CollectionName - }; - } -} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs deleted file mode 100644 index cdfaae18e5f2..000000000000 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizableTextSearchMetadata.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -[Experimental("SKEXP0020")] -public class VectorizableTextSearchMetadata -{ - /// The name of the vector store. - /// - /// Where possible, this maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems, see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// The name of the database. - /// - public string? DatabaseName { get; init; } - - /// - /// The name of a collection (table, container) within the database. - /// - public string? CollectionName { get; init; } - - /// - /// Initializes an instance of from . - /// - /// Instance of . - public static VectorizableTextSearchMetadata From(VectorStoreRecordCollectionMetadata collectionMetadata) - { - return new() - { - VectorStoreSystemName = collectionMetadata.VectorStoreSystemName, - DatabaseName = collectionMetadata.DatabaseName, - CollectionName = collectionMetadata.CollectionName - }; - } -} diff --git a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs b/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs deleted file mode 100644 index c6e7781c2925..000000000000 --- a/dotnet/src/Connectors/VectorData.Abstractions/VectorSearch/VectorizedSearchMetadata.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Diagnostics.CodeAnalysis; - -namespace Microsoft.Extensions.VectorData; - -/// Provides metadata about an . -[Experimental("SKEXP0020")] -public class VectorizedSearchMetadata -{ - /// The name of the vector store. - /// - /// Where possible, this maps to the "db.system.name" attribute defined in the - /// OpenTelemetry Semantic Conventions for database calls and systems, see . - /// Example: redis, sqlite, mysql. - /// - public string? VectorStoreSystemName { get; init; } - - /// - /// The name of the database. - /// - public string? DatabaseName { get; init; } - - /// - /// The name of a collection (table, container) within the database. - /// - public string? CollectionName { get; init; } - - /// - /// Initializes an instance of from . - /// - /// Instance of . - public static VectorizedSearchMetadata From(VectorStoreRecordCollectionMetadata collectionMetadata) - { - return new() - { - VectorStoreSystemName = collectionMetadata.VectorStoreSystemName, - DatabaseName = collectionMetadata.DatabaseName, - CollectionName = collectionMetadata.CollectionName - }; - } -} From 21dcf2ab67a0f148b7e5b7b382115444082a01f7 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 25 Mar 2025 09:06:53 -0700 Subject: [PATCH 17/19] Added OpenTelemetry decorators --- .../VectorData/OpenTelemetryConstants.cs | 43 +++ .../Connectors/VectorData/TelemetryHelpers.cs | 261 ++++++++++++++++++ .../OpenTelemetryKeywordHybridSearch.cs | 98 +++++++ ...tryKeywordHybridSearchBuilderExtensions.cs | 31 +++ .../OpenTelemetryVectorizableTextSearch.cs | 100 +++++++ ...VectorizableTextSearchBuilderExtensions.cs | 31 +++ .../OpenTelemetryVectorizedSearch.cs | 100 +++++++ ...emetryVectorizedSearchBuilderExtensions.cs | 31 +++ .../VectorStorage/OpenTelemetryVectorStore.cs | 104 +++++++ ...enTelemetryVectorStoreBuilderExtensions.cs | 31 +++ ...penTelemetryVectorStoreRecordCollection.cs | 257 +++++++++++++++++ ...rStoreRecordCollectionBuilderExtensions.cs | 32 +++ 12 files changed, 1119 insertions(+) create mode 100644 dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs create mode 100644 dotnet/src/Connectors/VectorData/TelemetryHelpers.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs create mode 100644 dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs diff --git a/dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs b/dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs new file mode 100644 index 000000000000..f053bba48d44 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/OpenTelemetryConstants.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Extensions.VectorData; + +/// +/// OpenTelemetry constants from Semantic Conventions for database calls and systems: +/// . +/// . +/// . +/// +internal static class OpenTelemetryConstants +{ + public const string DefaultSourceName = "Experimental.Microsoft.Extensions.VectorData"; + + public const string DbSystemName = "db.system.name"; + public const string DbCollectionName = "db.collection.name"; + public const string DbNamespace = "db.namespace"; + public const string DbOperationName = "db.operation.name"; + + public const string DbOperationDurationMetricName = "db.client.operation.duration"; + public const string DbOperationDurationMetricDescription = "Duration of database client operations."; + + public const string ErrorType = "error.type"; + + public const string SecondsUnit = "s"; + + public static string GetSourceNameOrDefault(string? sourceName) => !string.IsNullOrWhiteSpace(sourceName) ? sourceName! : DefaultSourceName; + + public static string GetActivityName(string operationName, string? collectionName, string? namespaceName) + { + if (!string.IsNullOrWhiteSpace(collectionName)) + { + return $"{operationName} {collectionName}"; + } + + if (!string.IsNullOrWhiteSpace(namespaceName)) + { + return $"{operationName} {namespaceName}"; + } + + return operationName; + } +} diff --git a/dotnet/src/Connectors/VectorData/TelemetryHelpers.cs b/dotnet/src/Connectors/VectorData/TelemetryHelpers.cs new file mode 100644 index 000000000000..2824f67c608c --- /dev/null +++ b/dotnet/src/Connectors/VectorData/TelemetryHelpers.cs @@ -0,0 +1,261 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// +/// Class with helper methods to run operations with telemetry. +/// +internal static class TelemetryHelpers +{ + /// + /// Method to run operation which doesn't return a result. + /// + internal static async Task RunOperationAsync( + ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Func operation) + { + using var activity = activitySource.StartActivity(operationName, collectionName, databaseName, vectorStoreSystemName); + Stopwatch? stopwatch = operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + Exception? exception = null; + + try + { + await operation().ConfigureAwait(false); + } + catch (Exception ex) + { + exception = ex; + throw; + } + finally + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + exception, + stopwatch); + } + } + + /// + /// Method to run operation which returns the result. + /// + internal static async Task RunOperationAsync( + ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Func> operation) + { + using var activity = activitySource.StartActivity(operationName, collectionName, databaseName, vectorStoreSystemName); + Stopwatch? stopwatch = operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + Exception? exception = null; + + try + { + return await operation().ConfigureAwait(false); + } + catch (Exception ex) + { + exception = ex; + throw; + } + finally + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + exception, + stopwatch); + } + } + + /// + /// Method to run operation which returns async enumeration. + /// + internal static async IAsyncEnumerable RunOperationAsync( + ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Func> operation, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + using var activity = activitySource.StartActivity(operationName, collectionName, databaseName, vectorStoreSystemName); + Stopwatch? stopwatch = operationDurationHistogram.Enabled ? Stopwatch.StartNew() : null; + + IAsyncEnumerator enumerator; + + try + { + enumerator = operation().GetAsyncEnumerator(cancellationToken); + } + catch (Exception ex) + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + ex, + stopwatch); + + throw; + } + + Exception? exception = null; + + try + { + while (true) + { + try + { + if (!await enumerator.MoveNextAsync().ConfigureAwait(false)) + { + break; + } + } + catch (Exception ex) + { + exception = ex; + throw; + } + + yield return enumerator.Current; + Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802 + } + } + finally + { + TelemetryHelpers.TraceOperationCompletion( + operationName, + collectionName, + databaseName, + vectorStoreSystemName, + operationDurationHistogram, + activity, + exception, + stopwatch); + + await enumerator.DisposeAsync().ConfigureAwait(false); + } + } + + #region private + + private static Activity? StartActivity( + this ActivitySource activitySource, + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName) + { + Activity? activity = null; + + if (activitySource.HasListeners()) + { + activity = activitySource.StartActivity( + OpenTelemetryConstants.GetActivityName(operationName, collectionName, databaseName), + ActivityKind.Client); + + var tags = GetTags(operationName, collectionName, databaseName, vectorStoreSystemName); + + foreach (var tag in tags) + { + activity?.SetTag(tag.Key, tag.Value); + } + } + + return activity; + } + + private static void TraceOperationCompletion( + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName, + Histogram operationDurationHistogram, + Activity? activity, + Exception? exception, + Stopwatch? stopwatch) + { + if (operationDurationHistogram.Enabled && stopwatch is not null) + { + var tags = TelemetryHelpers.GetTags(operationName, collectionName, databaseName, vectorStoreSystemName); + + if (exception is not null) + { + tags.Add(OpenTelemetryConstants.ErrorType, exception.GetType().FullName); + } + + operationDurationHistogram.Record(stopwatch.Elapsed.TotalSeconds, tags); + } + + if (exception is not null) + { + activity? + .SetTag(OpenTelemetryConstants.ErrorType, exception.GetType().FullName) + .SetStatus(ActivityStatusCode.Error, exception.Message); + } + } + + private static TagList GetTags( + string operationName, + string? collectionName, + string? databaseName, + string? vectorStoreSystemName) + { + TagList tags = default; + + tags.Add(OpenTelemetryConstants.DbOperationName, operationName); + + if (!string.IsNullOrWhiteSpace(collectionName)) + { + tags.Add(OpenTelemetryConstants.DbCollectionName, collectionName); + } + + if (!string.IsNullOrWhiteSpace(databaseName)) + { + tags.Add(OpenTelemetryConstants.DbNamespace, databaseName); + } + + if (!string.IsNullOrWhiteSpace(vectorStoreSystemName)) + { + tags.Add(OpenTelemetryConstants.DbSystemName, vectorStoreSystemName); + } + + return tags; + } + + #endregion +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs new file mode 100644 index 000000000000..3c4aa653dcc2 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearch.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating search that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryKeywordHybridSearch : IKeywordHybridSearch, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IKeywordHybridSearch _innerSearch; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryKeywordHybridSearch(IKeywordHybridSearch innerSearch, string? sourceName = null) + { + this._innerSearch = innerSearch ?? throw new ArgumentNullException(nameof(innerSearch)); + + if (this._innerSearch.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new(telemetrySourceName); + this._meter = new(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public Task> HybridSearchAsync(TVector vector, ICollection keywords, HybridSearchOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "hybrid_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerSearch.HybridSearchAsync(vector, keywords, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = serviceType ?? throw new ArgumentNullException(nameof(serviceType)); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs new file mode 100644 index 000000000000..87445776f8e9 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryKeywordHybridSearchBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryKeywordHybridSearchBuilderExtensions +{ + /// Adds OpenTelemetry support to the keyword hybrid search pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static KeywordHybridSearchBuilder UseOpenTelemetry( + this KeywordHybridSearchBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerSearch, services) => + { + return new OpenTelemetryKeywordHybridSearch(innerSearch, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs new file mode 100644 index 000000000000..f9b2cc8397dc --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearch.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vectorizable text search that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryVectorizableTextSearch : IVectorizableTextSearch, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorizableTextSearch _innerSearch; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorizableTextSearch(IVectorizableTextSearch innerSearch, string? sourceName = null) + { + this._innerSearch = innerSearch ?? throw new ArgumentNullException(nameof(innerSearch)); + + if (this._innerSearch.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(telemetrySourceName); + this._meter = new Meter(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public Task> VectorizableTextSearchAsync( + string searchText, + VectorSearchOptions? options = default, + CancellationToken cancellationToken = default) + { + const string OperationName = "vectorizable_text_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerSearch.VectorizableTextSearchAsync(searchText, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = serviceType ?? throw new ArgumentNullException(nameof(serviceType)); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs new file mode 100644 index 000000000000..4f89fc2fcc97 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizableTextSearchBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorizableTextSearchBuilderExtensions +{ + /// Adds OpenTelemetry support to the vectorizable text search pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorizableTextSearchBuilder UseOpenTelemetry( + this VectorizableTextSearchBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerSearch, services) => + { + return new OpenTelemetryVectorizableTextSearch(innerSearch, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs new file mode 100644 index 000000000000..7896b5664546 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearch.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vectorized search that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryVectorizedSearch : IVectorizedSearch, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorizedSearch _innerSearch; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorizedSearch(IVectorizedSearch innerSearch, string? sourceName = null) + { + this._innerSearch = innerSearch ?? throw new ArgumentNullException(nameof(innerSearch)); + + if (this._innerSearch.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(telemetrySourceName); + this._meter = new Meter(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public Task> VectorizedSearchAsync( + TVector vector, + VectorSearchOptions? options = default, + CancellationToken cancellationToken = default) + { + const string OperationName = "vectorized_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerSearch.VectorizedSearchAsync(vector, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + _ = serviceType ?? throw new ArgumentNullException(nameof(serviceType)); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerSearch.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs new file mode 100644 index 000000000000..56390ff4305c --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorSearch/OpenTelemetryVectorizedSearchBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorizedSearchBuilderExtensions +{ + /// Adds OpenTelemetry support to the vectorized search pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorizedSearchBuilder UseOpenTelemetry( + this VectorizedSearchBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerSearch, services) => + { + return new OpenTelemetryVectorizedSearch(innerSearch, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs new file mode 100644 index 000000000000..edf647203da4 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStore.cs @@ -0,0 +1,104 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vector store that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +public class OpenTelemetryVectorStore : IVectorStore, IDisposable +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorStore _innerStore; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string _telemetrySourceName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorStore(IVectorStore innerStore, string? sourceName = null) + { + this._innerStore = innerStore ?? throw new ArgumentNullException(nameof(innerStore)); + + if (this._innerStore.GetService(typeof(VectorStoreMetadata)) is VectorStoreMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + } + + this._telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(this._telemetrySourceName); + this._meter = new Meter(this._telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + public IVectorStoreRecordCollection GetCollection(string name, VectorStoreRecordDefinition? vectorStoreRecordDefinition = null) where TKey : notnull + => new OpenTelemetryVectorStoreRecordCollection( + this._innerStore.GetCollection(name, vectorStoreRecordDefinition), + this._telemetrySourceName); + + /// + public IAsyncEnumerable ListCollectionNamesAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "list_collection_names"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + null, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerStore.ListCollectionNamesAsync(cancellationToken), + cancellationToken); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerStore.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs new file mode 100644 index 000000000000..5f47d0708349 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreBuilderExtensions.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorStoreBuilderExtensions +{ + /// Adds OpenTelemetry support to the vector store pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorStoreBuilder UseOpenTelemetry( + this VectorStoreBuilder builder, + string? sourceName = null) + { + Verify.NotNull(builder); + + return builder.Use((innerStore, services) => + { + return new OpenTelemetryVectorStore(innerStore, sourceName); + }); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs new file mode 100644 index 000000000000..bf09d4c3f501 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs @@ -0,0 +1,257 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Metrics; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Represents a delegating vector store record collection that implements the OpenTelemetry Semantic Conventions for database calls and systems. +/// +/// This class provides an implementation of the Semantic Conventions for database calls and systems v1.31, defined at . +/// The part of the specification is still experimental and subject to change; as such, the telemetry output by this class is also subject to change. +/// +[Experimental("SKEXP0020")] +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public class OpenTelemetryVectorStoreRecordCollection : IVectorStoreRecordCollection, IDisposable where TKey : notnull +#pragma warning restore CA1711 // Identifiers should not have incorrect suffix +{ + private readonly ActivitySource _activitySource; + private readonly Meter _meter; + private readonly Histogram _operationDurationHistogram; + private readonly IVectorStoreRecordCollection _innerCollection; + private readonly string? _vectorStoreSystemName; + private readonly string? _databaseName; + private readonly string? _collectionName; + + /// + /// Initializes a new instance of the class. + /// + /// The underlying . + /// An optional source name that will be used on the telemetry data. + public OpenTelemetryVectorStoreRecordCollection(IVectorStoreRecordCollection innerCollection, string? sourceName = null) + { + this._innerCollection = innerCollection ?? throw new ArgumentNullException(nameof(innerCollection)); + + if (this._innerCollection.GetService(typeof(VectorStoreRecordCollectionMetadata)) is VectorStoreRecordCollectionMetadata metadata) + { + this._vectorStoreSystemName = metadata.VectorStoreSystemName; + this._databaseName = metadata.DatabaseName; + this._collectionName = metadata.CollectionName; + } + + var telemetrySourceName = OpenTelemetryConstants.GetSourceNameOrDefault(sourceName); + + this._activitySource = new ActivitySource(telemetrySourceName); + this._meter = new Meter(telemetrySourceName); + + this._operationDurationHistogram = this._meter.CreateHistogram( + OpenTelemetryConstants.DbOperationDurationMetricName, + OpenTelemetryConstants.SecondsUnit, + OpenTelemetryConstants.DbOperationDurationMetricDescription); + } + + /// + [Obsolete("Use GetService(typeof(VectorStoreRecordCollectionMetadata)) to get an information about vector store record collection.")] + public string CollectionName => this._innerCollection.CollectionName; + + /// + public Task CollectionExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "collection_exists"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.CollectionExistsAsync(cancellationToken)); + } + + /// + public Task CreateCollectionAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "create_collection"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.CreateCollectionAsync(cancellationToken)); + } + + /// + public Task CreateCollectionIfNotExistsAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "collection_exists_if_not_exists"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.CreateCollectionIfNotExistsAsync(cancellationToken)); + } + + /// + public Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) + { + const string OperationName = "delete"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.DeleteAsync(key, cancellationToken)); + } + + /// + public Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default) + { + const string OperationName = "delete_batch"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.DeleteBatchAsync(keys, cancellationToken)); + } + + /// + public Task DeleteCollectionAsync(CancellationToken cancellationToken = default) + { + const string OperationName = "delete_collection"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.DeleteCollectionAsync(cancellationToken)); + } + + /// + public Task GetAsync(TKey key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "get"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.GetAsync(key, options, cancellationToken)); + } + + /// + public IAsyncEnumerable GetBatchAsync(IEnumerable keys, GetRecordOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "get_batch"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.GetBatchAsync(keys, options, cancellationToken), + cancellationToken); + } + + /// + public Task UpsertAsync(TRecord record, CancellationToken cancellationToken = default) + { + const string OperationName = "upsert"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.UpsertAsync(record, cancellationToken)); + } + + /// + public IAsyncEnumerable UpsertBatchAsync(IEnumerable records, CancellationToken cancellationToken = default) + { + const string OperationName = "upsert_batch"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.UpsertBatchAsync(records, cancellationToken), + cancellationToken); + } + + /// + public Task> VectorizedSearchAsync(TVector vector, VectorSearchOptions? options = null, CancellationToken cancellationToken = default) + { + const string OperationName = "vectorized_search"; + + return TelemetryHelpers.RunOperationAsync( + this._activitySource, + OperationName, + this._collectionName, + this._databaseName, + this._vectorStoreSystemName, + this._operationDurationHistogram, + () => this._innerCollection.VectorizedSearchAsync(vector, options, cancellationToken)); + } + + /// + public object? GetService(Type serviceType, object? serviceKey = null) + { + Verify.NotNull(serviceType); + + return + serviceKey is null && serviceType.IsInstanceOfType(this) ? this : + this._innerCollection.GetService(serviceType, serviceKey); + } + + /// + public void Dispose() + { + this.Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + /// Provides a mechanism for releasing unmanaged resources. + /// if being called from ; otherwise, . + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + this._activitySource.Dispose(); + this._meter.Dispose(); + } + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs new file mode 100644 index 000000000000..06bb83404570 --- /dev/null +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollectionBuilderExtensions.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.SemanticKernel; + +namespace Microsoft.Extensions.VectorData; + +/// Provides extensions for configuring instances. +[Experimental("SKEXP0020")] +public static class OpenTelemetryVectorStoreRecordCollectionBuilderExtensions +{ + /// Adds OpenTelemetry support to the vector store record collection pipeline, following the OpenTelemetry Semantic Conventions for database calls and systems. + /// + /// The draft specification this follows is available at . + /// The part of the specification is still experimental and subject to change; as such, the telemetry output is also subject to change. + /// + /// The . + /// An optional source name that will be used on the telemetry data. + /// The . + public static VectorStoreRecordCollectionBuilder UseOpenTelemetry( + this VectorStoreRecordCollectionBuilder builder, + string? sourceName = null) + where TKey : notnull + { + Verify.NotNull(builder); + + return builder.Use((innerCollection, services) => + { + return new OpenTelemetryVectorStoreRecordCollection(innerCollection, sourceName); + }); + } +} From bcf5483500e15f595c3cde895d5fdda841dae68e Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 25 Mar 2025 12:31:30 -0700 Subject: [PATCH 18/19] Added unit tests --- dotnet/Directory.Packages.props | 1 + .../TelemetryHelpersTests.cs | 433 ++++++++++++++++++ .../VectorData.UnitTests.csproj | 2 +- .../OpenTelemetryKeywordHybridSearchTests.cs | 67 +++ ...penTelemetryVectorizableTextSearchTests.cs | 65 +++ .../OpenTelemetryVectorizedSearchTests.cs | 67 +++ ...lemetryVectorStoreRecordCollectionTests.cs | 418 +++++++++++++++++ .../OpenTelemetryVectorStoreTests.cs | 60 +++ .../Connectors/VectorData/VectorData.csproj | 4 + ...penTelemetryVectorStoreRecordCollection.cs | 2 +- 10 files changed, 1117 insertions(+), 2 deletions(-) create mode 100644 dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs create mode 100644 dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs create mode 100644 dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs create mode 100644 dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs create mode 100644 dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs create mode 100644 dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 290c5501bc07..b2a97f798413 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -59,6 +59,7 @@ + diff --git a/dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs new file mode 100644 index 000000000000..aa819b4b74bc --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/TelemetryHelpersTests.cs @@ -0,0 +1,433 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests; + +public class TelemetryHelpersTests +{ + [Fact] + public async Task RunOperationWithoutResultWorksWithActivityAndMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + bool operationExecuted = false; + async Task Operation() + { + await Task.Yield(); + operationExecuted = true; + } + + // Act + await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation); + + // Assert + Assert.True(operationExecuted); + + var activity = Assert.Single(activities); + + Assert.Equal($"{operationName} {collectionName}", activity.DisplayName); + Assert.Equal(operationName, activity.GetTagItem("db.operation.name")); + Assert.Equal(collectionName, activity.GetTagItem("db.collection.name")); + Assert.Equal(databaseName, activity.GetTagItem("db.namespace")); + Assert.Equal(systemName, activity.GetTagItem("db.system.name")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + Assert.Equal("db.client.operation.duration", metric.Name); + + this.AssertMetric(metric, operationName); + } + + [Fact] + public async Task RunOperationWithoutResultRecordsErrorOnExceptionAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + static Task Operation() => throw new InvalidOperationException("Test exception"); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => + TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation)); + + Assert.Equal("Test exception", exception.Message); + + var activity = Assert.Single(activities); + + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + this.AssertMetricError(metric); + } + + [Fact] + public async Task RunOperationWithResultWorksWithActivityAndMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + async static Task Operation() + { + await Task.Yield(); + return 42; + } + + // Act + var result = await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation); + + // Assert + Assert.Equal(42, result); + + var activity = Assert.Single(activities); + Assert.Equal($"{operationName} {collectionName}", activity.DisplayName); + Assert.Equal(operationName, activity.GetTagItem("db.operation.name")); + Assert.Equal(collectionName, activity.GetTagItem("db.collection.name")); + Assert.Equal(databaseName, activity.GetTagItem("db.namespace")); + Assert.Equal(systemName, activity.GetTagItem("db.system.name")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + Assert.Equal("db.client.operation.duration", metric.Name); + + this.AssertMetric(metric, operationName); + } + + [Fact] + public async Task RunOperationWithResultRecordsErrorOnExceptionAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + static Task Operation() => throw new InvalidOperationException("Test exception"); + + // Act & Assert + var exception = await Assert.ThrowsAsync(() => + TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation)); + + Assert.Equal("Test exception", exception.Message); + + var activity = Assert.Single(activities); + + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + this.AssertMetricError(metric); + } + + [Fact] + public async Task RunOperationWithAsyncEnumerableWorksWithActivityAndMetricsAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var result = new[] { 1, 2, 3 }; + + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + IAsyncEnumerable Operation() + { + return result.ToAsyncEnumerable(); + } + + // Act + var results = await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation, + default).ToListAsync(); + + // Assert + Assert.Equal(result, results); + + var activity = Assert.Single(activities); + + Assert.Equal($"{operationName} {collectionName}", activity.DisplayName); + Assert.Equal(operationName, activity.GetTagItem("db.operation.name")); + Assert.Equal(collectionName, activity.GetTagItem("db.collection.name")); + Assert.Equal(databaseName, activity.GetTagItem("db.namespace")); + Assert.Equal(systemName, activity.GetTagItem("db.system.name")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + Assert.Equal("db.client.operation.duration", metric.Name); + + this.AssertMetric(metric, operationName); + } + + [Fact] + public async Task RunOperationWithAsyncEnumerableRecordsErrorOnExceptionDuringEnumerationAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var operationName = "test_operation"; + var collectionName = "testcollection"; + var databaseName = "testdb"; + var systemName = "testsystem"; + + using var activitySource = new ActivitySource(sourceName); + using var meter = new Meter(sourceName); + var histogram = meter.CreateHistogram("db.client.operation.duration", "s", "Duration of database client operations"); + + var activities = new List(); + var metrics = new List(); + + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + using var meterProvider = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .AddMeter(sourceName) + .AddInMemoryExporter(metrics) + .Build(); + + async static IAsyncEnumerable Operation() + { + yield return 1; + await Task.Yield(); + throw new InvalidOperationException("Test exception"); + } + + // Act & Assert + var exception = await Assert.ThrowsAsync(async () => + await TelemetryHelpers.RunOperationAsync( + activitySource, + operationName, + collectionName, + databaseName, + systemName, + histogram, + Operation, + default).ToListAsync()); + + Assert.Equal("Test exception", exception.Message); + + var activity = Assert.Single(activities); + + Assert.Equal(ActivityStatusCode.Error, activity.Status); + Assert.Equal(typeof(InvalidOperationException).FullName, activity.GetTagItem("error.type")); + + // Force metrics to be collected + meterProvider.ForceFlush(timeoutMilliseconds: 1000); + + var metric = Assert.Single(metrics); + + this.AssertMetricError(metric); + } + + #region private + + private void AssertMetric(Metric? metric, string operationName) + { + var metricExists = false; + + if (metric is not null) + { + foreach (var point in metric.GetMetricPoints()) + { + var histogramExists = point.GetHistogramSum() > 0; + var tagExists = false; + + foreach (var tag in point.Tags) + { + if (tag.Key == "db.operation.name" && tag.Value?.ToString() == operationName) + { + tagExists = true; + } + } + + metricExists = histogramExists && tagExists; + } + } + + Assert.True(metricExists); + } + + private void AssertMetricError(Metric? metric) + { + var metricErrorExists = false; + + if (metric is not null) + { + foreach (var point in metric.GetMetricPoints()) + { + foreach (var tag in point.Tags) + { + metricErrorExists = + tag.Key == "error.type" && + tag.Value?.ToString() == typeof(InvalidOperationException).FullName; + } + } + } + + Assert.True(metricErrorExists); + } + + #endregion +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj b/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj index d374de2022ba..4cde8a374c10 100644 --- a/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorData.UnitTests.csproj @@ -12,6 +12,7 @@ + @@ -29,7 +30,6 @@ - diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs new file mode 100644 index 000000000000..b538ac2b9534 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryKeywordHybridSearchTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorSearch; + +public class OpenTelemetryKeywordHybridSearchTests +{ + [Fact] + public async Task HybridSearchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List> { new("record1", 0.95f) }.ToAsyncEnumerable()); + + var mockInnerSearch = new Mock>(); + + mockInnerSearch + .Setup(s => s.HybridSearchAsync(It.IsAny(), It.IsAny>(), null, default)) + .ReturnsAsync(expectedResults); + + mockInnerSearch + .Setup(s => s.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + + var innerSearch = mockInnerSearch.Object; + var keywordHybridSearch = innerSearch + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var vector = new float[] { 1.0f }; + + // Act + var result = await keywordHybridSearch.HybridSearchAsync(vector, ["keyword1"]); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("hybrid_search testcollection", activity.DisplayName); + Assert.Equal("hybrid_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs new file mode 100644 index 000000000000..7fdbb3eb6da0 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizableTextSearchTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorSearch; + +public class OpenTelemetryVectorizableTextSearchTests +{ + [Fact] + public async Task VectorizableTextSearchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List> { new("record1", 0.90f) }.ToAsyncEnumerable()); + + var mockInnerSearch = new Mock>(); + + mockInnerSearch + .Setup(s => s.VectorizableTextSearchAsync("test query", null, default)) + .ReturnsAsync(expectedResults); + + mockInnerSearch + .Setup(s => s.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + + var innerSearch = mockInnerSearch.Object; + var vectorizableTextSearch = innerSearch + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorizableTextSearch.VectorizableTextSearchAsync("test query"); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("vectorizable_text_search testcollection", activity.DisplayName); + Assert.Equal("vectorizable_text_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs new file mode 100644 index 000000000000..28c74596a48e --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorSearch/OpenTelemetryVectorizedSearchTests.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorSearch; + +public class OpenTelemetryVectorizedSearchTests +{ + [Fact] + public async Task VectorizedSearchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List> { new("record1", 0.95f) }.ToAsyncEnumerable()); + + var mockInnerSearch = new Mock>(); + + mockInnerSearch + .Setup(s => s.VectorizedSearchAsync(It.IsAny(), null, default)) + .ReturnsAsync(expectedResults); + + mockInnerSearch + .Setup(s => s.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + + var innerSearch = mockInnerSearch.Object; + var vectorizedSearch = innerSearch + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var vector = new float[] { 1.0f, 2.0f }; + + // Act + var result = await vectorizedSearch.VectorizedSearchAsync(vector); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("vectorized_search testcollection", activity.DisplayName); + Assert.Equal("vectorized_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs new file mode 100644 index 000000000000..90c6f2569294 --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreRecordCollectionTests.cs @@ -0,0 +1,418 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorStorage; + +public class OpenTelemetryVectorStoreRecordCollectionTests +{ + private readonly Mock> _mockCollection; + + public OpenTelemetryVectorStoreRecordCollectionTests() + { + this._mockCollection = new(); + + this._mockCollection.Setup(c => c.GetService(typeof(VectorStoreRecordCollectionMetadata), It.IsAny())) + .Returns(new VectorStoreRecordCollectionMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb", + CollectionName = "testcollection" + }); + } + + [Fact] + public async Task CollectionExistsWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.CollectionExistsAsync(default)).ReturnsAsync(true); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.CollectionExistsAsync(); + + // Assert + Assert.True(result); + + var activity = Assert.Single(activities); + + Assert.Equal("collection_exists testcollection", activity.DisplayName); + Assert.Equal("collection_exists", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task CreateCollectionWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.CreateCollectionAsync(default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.CreateCollectionAsync(); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("create_collection testcollection", activity.DisplayName); + Assert.Equal("create_collection", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task CreateCollectionIfNotExistsWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.CreateCollectionIfNotExistsAsync(default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.CreateCollectionIfNotExistsAsync(); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("create_collection_if_not_exists testcollection", activity.DisplayName); + Assert.Equal("create_collection_if_not_exists", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task DeleteCollectionWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.DeleteCollectionAsync(default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.DeleteCollectionAsync(); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("delete_collection testcollection", activity.DisplayName); + Assert.Equal("delete_collection", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task GetWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedRecord = new object(); + + this._mockCollection.Setup(c => c.GetAsync("key1", null, default)).ReturnsAsync(expectedRecord); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.GetAsync("key1"); + + // Assert + Assert.Equal(expectedRecord, result); + + var activity = Assert.Single(activities); + + Assert.Equal("get testcollection", activity.DisplayName); + Assert.Equal("get", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task GetBatchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + object[] expectedRecords = [new(), new()]; + string[] keys = ["key1", "key2"]; + + this._mockCollection.Setup(c => c.GetBatchAsync(keys, null, default)) + .Returns(expectedRecords.ToAsyncEnumerable()); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.GetBatchAsync(keys).ToListAsync(); + + // Assert + Assert.Equal(expectedRecords, result); + + var activity = Assert.Single(activities); + + Assert.Equal("get_batch testcollection", activity.DisplayName); + Assert.Equal("get_batch", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task DeleteWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + this._mockCollection.Setup(c => c.DeleteAsync("key1", default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.DeleteAsync("key1"); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("delete testcollection", activity.DisplayName); + Assert.Equal("delete", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task DeleteBatchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + string[] keys = ["key1", "key2"]; + + this._mockCollection.Setup(c => c.DeleteBatchAsync(keys, default)).Returns(Task.CompletedTask); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + await vectorStoreCollection.DeleteBatchAsync(keys); + + // Assert + var activity = Assert.Single(activities); + + Assert.Equal("delete_batch testcollection", activity.DisplayName); + Assert.Equal("delete_batch", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task UpsertWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedKey = "key1"; + + this._mockCollection.Setup(c => c.UpsertAsync(It.IsAny(), default)).ReturnsAsync(expectedKey); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.UpsertAsync(new object()); + + // Assert + Assert.Equal(expectedKey, result); + + var activity = Assert.Single(activities); + + Assert.Equal("upsert testcollection", activity.DisplayName); + Assert.Equal("upsert", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task UpsertBatchWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + string[] expectedKeys = ["key1", "key2"]; + + this._mockCollection.Setup(c => c.UpsertBatchAsync(It.IsAny>(), default)) + .Returns(expectedKeys.ToAsyncEnumerable()); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStoreCollection.UpsertBatchAsync([new object(), new object()]).ToListAsync(); + + // Assert + Assert.Equal(expectedKeys, result); + + var activity = Assert.Single(activities); + + Assert.Equal("upsert_batch testcollection", activity.DisplayName); + Assert.Equal("upsert_batch", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } + + [Fact] + public async Task VectorizedSearchAsyncWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + var expectedResults = new VectorSearchResults(new List>().ToAsyncEnumerable()); + + this._mockCollection.Setup(c => c.VectorizedSearchAsync(It.IsAny(), null, default)) + .ReturnsAsync(expectedResults); + + var innerCollection = this._mockCollection.Object; + var vectorStoreCollection = innerCollection + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + var vector = new float[] { 1.0f }; + + // Act + var result = await vectorStoreCollection.VectorizedSearchAsync(vector); + + // Assert + Assert.Equal(expectedResults, result); + + var activity = Assert.Single(activities); + + Assert.Equal("vectorized_search testcollection", activity.DisplayName); + Assert.Equal("vectorized_search", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testcollection", activity.GetTagItem("db.collection.name")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs new file mode 100644 index 000000000000..4ab0a1bb8b5c --- /dev/null +++ b/dotnet/src/Connectors/VectorData.UnitTests/VectorStorage/OpenTelemetryVectorStoreTests.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Extensions.VectorData; +using Moq; +using OpenTelemetry.Trace; +using Xunit; + +namespace VectorData.UnitTests.VectorStorage; + +public class OpenTelemetryVectorStoreTests +{ + [Fact] + public async Task ListCollectionNamesWorksWithActivityAsync() + { + // Arrange + var sourceName = Guid.NewGuid().ToString(); + + string[] colletions = ["col1", "col2"]; + var mockInnerStore = new Mock(); + + mockInnerStore.Setup(s => s.ListCollectionNamesAsync(default)) + .Returns(colletions.ToAsyncEnumerable()); + + mockInnerStore.Setup(s => s.GetService(typeof(VectorStoreMetadata), It.IsAny())) + .Returns(new VectorStoreMetadata + { + VectorStoreSystemName = "testvectorstore", + DatabaseName = "testdb" + }); + + var vectorStore = mockInnerStore.Object + .AsBuilder() + .UseOpenTelemetry(sourceName) + .Build(); + + var activities = new List(); + using var tracerProvider = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .AddSource(sourceName) + .AddInMemoryExporter(activities) + .Build(); + + // Act + var result = await vectorStore.ListCollectionNamesAsync().ToListAsync(); + + // Assert + Assert.Equal(colletions, result); + + var activity = Assert.Single(activities); + + Assert.Equal("list_collection_names testdb", activity.DisplayName); + Assert.Equal("list_collection_names", activity.GetTagItem("db.operation.name")); + Assert.Equal("testdb", activity.GetTagItem("db.namespace")); + Assert.Equal("testvectorstore", activity.GetTagItem("db.system.name")); + } +} diff --git a/dotnet/src/Connectors/VectorData/VectorData.csproj b/dotnet/src/Connectors/VectorData/VectorData.csproj index 31e84c2533ce..f80443a6c310 100644 --- a/dotnet/src/Connectors/VectorData/VectorData.csproj +++ b/dotnet/src/Connectors/VectorData/VectorData.csproj @@ -59,4 +59,8 @@ + + + + diff --git a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs index bf09d4c3f501..830939fec0e4 100644 --- a/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/VectorData/VectorStorage/OpenTelemetryVectorStoreRecordCollection.cs @@ -93,7 +93,7 @@ public Task CreateCollectionAsync(CancellationToken cancellationToken = default) /// public Task CreateCollectionIfNotExistsAsync(CancellationToken cancellationToken = default) { - const string OperationName = "collection_exists_if_not_exists"; + const string OperationName = "create_collection_if_not_exists"; return TelemetryHelpers.RunOperationAsync( this._activitySource, From 27dd0c2c2aefb2b662ae59524644fb99b4e74930 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 25 Mar 2025 13:24:19 -0700 Subject: [PATCH 19/19] Added tracing and metering examples --- dotnet/samples/Concepts/Concepts.csproj | 2 + .../Concepts/Memory/VectorStore_Telemetry.cs | 151 +++++++++++++++++- 2 files changed, 152 insertions(+), 1 deletion(-) diff --git a/dotnet/samples/Concepts/Concepts.csproj b/dotnet/samples/Concepts/Concepts.csproj index 0a8add50d2d2..31d6a392de0e 100644 --- a/dotnet/samples/Concepts/Concepts.csproj +++ b/dotnet/samples/Concepts/Concepts.csproj @@ -14,6 +14,7 @@ + @@ -22,6 +23,7 @@ + diff --git a/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs b/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs index 2165c96aabb2..ae72fd36c9d7 100644 --- a/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs +++ b/dotnet/samples/Concepts/Memory/VectorStore_Telemetry.cs @@ -1,12 +1,17 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Diagnostics; using Azure.Identity; +using Azure.Monitor.OpenTelemetry.Exporter; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.VectorData; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AzureOpenAI; using Microsoft.SemanticKernel.Connectors.InMemory; using Microsoft.SemanticKernel.Embeddings; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; namespace Memory; @@ -14,8 +19,13 @@ namespace Memory; /// A simple example showing how to ingest data into a vector store and then use vector search to find related records to a given string /// with enabled telemetry. /// -public class VectorStore_Telemetry(ITestOutputHelper output) : BaseTest(output) +public class VectorStore_Telemetry(ITestOutputHelper output) : BaseTest(output, redirectSystemConsoleOutput: true) { + /// + /// Instance of for the example's main activity. + /// + private static readonly ActivitySource s_activitySource = new("VectorStoreTelemetry.Example"); + [Fact] public async Task LoggingManualRegistrationAsync() { @@ -91,6 +101,143 @@ public async Task LoggingDependencyInjectionAsync() // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data. } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task TracingAsync(bool useApplicationInsights) + { + // Create an embedding generation service. + var textEmbeddingGenerationService = new AzureOpenAITextEmbeddingGenerationService( + TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, + TestConfiguration.AzureOpenAIEmbeddings.Endpoint, + new AzureCliCredential()); + + // Manually construct an InMemory vector store with enabled OpenTelemetry. + var vectorStore = new InMemoryVectorStore() + .AsBuilder() + .UseOpenTelemetry() + .Build(); + + using var tracerProvider = GetTracerProvider(useApplicationInsights); + + using var activity = s_activitySource.StartActivity("MainActivity"); + Console.WriteLine($"Operation/Trace ID: {Activity.Current?.TraceId}"); + + await RunExampleAsync(textEmbeddingGenerationService, vectorStore); + + // Output: + // Activity.DisplayName: create_collection_if_not_exists skglossary + // Activity.Duration: 00:00:00.0026442 + // Activity.Tags: + // db.operation.name: create_collection_if_not_exists + // db.collection.name: skglossary + // db.system.name: inmemory + // Activity.DisplayName: upsert skglossary + // Activity.Duration: 00:00:00.0010686 + // Activity.Tags: + // db.operation.name: upsert + // db.collection.name: skglossary + // db.system.name: inmemory + // and more... + + // Search string: What is an Application Programming Interface + // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data. + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task MeteringAsync(bool useApplicationInsights) + { + // Create an embedding generation service. + var textEmbeddingGenerationService = new AzureOpenAITextEmbeddingGenerationService( + TestConfiguration.AzureOpenAIEmbeddings.DeploymentName, + TestConfiguration.AzureOpenAIEmbeddings.Endpoint, + new AzureCliCredential()); + + // Manually construct an InMemory vector store with enabled OpenTelemetry. + var vectorStore = new InMemoryVectorStore() + .AsBuilder() + .UseOpenTelemetry() + .Build(); + + using var meterProvider = GetMeterProvider(useApplicationInsights); + + await RunExampleAsync(textEmbeddingGenerationService, vectorStore); + + // Output: + // Metric Name: db.client.operation.duration, Duration of database client operations., Unit: s, Meter: Experimental.Microsoft.Extensions.VectorData + + // (2025-03-25T20:03:17.9938116Z, 2025-03-25T20:03:20.2214978Z] db.collection.name: skglossary db.operation.name: create_collection_if_not_exists db.system.name: inmemory Histogram + // Value: Sum: 0.0015761 Count: 1 Min: 0.0015761 Max: 0.0015761 + + // (2025-03-25T20:03:17.9938116Z, 2025-03-25T20:03:20.2214978Z] db.collection.name: skglossary db.operation.name: upsert db.system.name: inmemory Histogram + // Value: Sum: 0.0011708 Count: 3 Min: 7E-06 Max: 0.0009944 + + // (2025-03-25T20:03:17.9938116Z, 2025-03-25T20:03:20.2214978Z] db.collection.name: skglossary db.operation.name: vectorized_search db.system.name: inmemory Histogram + // Value: Sum: 0.0109487 Count: 1 Min: 0.0109487 Max: 0.0109487 + + // Search string: What is an Application Programming Interface + // Result: Application Programming Interface. A set of rules and specifications that allow software components to communicate and exchange data. + } + + #region private + + private TracerProvider? GetTracerProvider(bool useApplicationInsights) + { + var tracerProviderBuilder = OpenTelemetry.Sdk.CreateTracerProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Vector Data Tracing Example")) + .AddSource("Experimental.Microsoft.Extensions.VectorData*") + .AddSource(s_activitySource.Name); + + if (useApplicationInsights) + { + var connectionString = TestConfiguration.ApplicationInsights.ConnectionString; + + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ConfigurationNotFoundException( + nameof(TestConfiguration.ApplicationInsights), + nameof(TestConfiguration.ApplicationInsights.ConnectionString)); + } + + tracerProviderBuilder.AddAzureMonitorTraceExporter(o => o.ConnectionString = connectionString); + } + else + { + tracerProviderBuilder.AddConsoleExporter(); + } + + return tracerProviderBuilder.Build(); + } + + private MeterProvider? GetMeterProvider(bool useApplicationInsights) + { + var meterProviderBuilder = OpenTelemetry.Sdk.CreateMeterProviderBuilder() + .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("Vector Data Metering Example")) + .AddMeter("Experimental.Microsoft.Extensions.VectorData*"); + + if (useApplicationInsights) + { + var connectionString = TestConfiguration.ApplicationInsights.ConnectionString; + + if (string.IsNullOrWhiteSpace(connectionString)) + { + throw new ConfigurationNotFoundException( + nameof(TestConfiguration.ApplicationInsights), + nameof(TestConfiguration.ApplicationInsights.ConnectionString)); + } + + meterProviderBuilder.AddAzureMonitorMetricExporter(o => o.ConnectionString = connectionString); + } + else + { + meterProviderBuilder.AddConsoleExporter(); + } + + return meterProviderBuilder.Build(); + } + private async Task RunExampleAsync( ITextEmbeddingGenerationService textEmbeddingGenerationService, IVectorStore vectorStore) @@ -177,4 +324,6 @@ private static IEnumerable CreateGlossaryEntries() Definition = "Retrieval Augmented Generation - a term that refers to the process of retrieving additional data to provide as context to an LLM to use when generating a response (completion) to a user’s question (prompt)." }; } + + #endregion }