From 1d2765bc1249f89fa8ae3a81dd0fe6d90e7bda8e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 27 Feb 2025 10:58:51 +0100 Subject: [PATCH 1/8] introduce a new project for integration tests of Pinecone that use local emulator behind the scenes --- dotnet/Directory.Packages.props | 1 + dotnet/SK-dotnet.sln | 21 +++-- .../PineconeIntegrationTests.csproj | 49 ++++++++++++ .../PineconeIntegrationTests/SampleTests.cs | 23 ++++++ .../Support/PineconeFixture.cs | 13 +++ .../Support/PineconeTestStore.cs | 79 +++++++++++++++++++ 6 files changed, 180 insertions(+), 6 deletions(-) create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index fcad75436cb8..21ff6dba6394 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -46,6 +46,7 @@ + diff --git a/dotnet/SK-dotnet.sln b/dotnet/SK-dotnet.sln index e1953ea0bf7e..85e900aa1b30 100644 --- a/dotnet/SK-dotnet.sln +++ b/dotnet/SK-dotnet.sln @@ -465,6 +465,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CosmosMongoDBIntegrationTes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureAISearchIntegrationTests", "src\VectorDataIntegrationTests\AzureAISearchIntegrationTests\AzureAISearchIntegrationTests.csproj", "{06181F0F-A375-43AE-B45F-73CBCFC30C14}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PineconeIntegrationTests", "src\VectorDataIntegrationTests\PineconeIntegrationTests\PineconeIntegrationTests.csproj", "{9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -1102,12 +1104,6 @@ Global {6F591D05-5F7F-4211-9042-42D8BCE60415}.Publish|Any CPU.Build.0 = Debug|Any CPU {6F591D05-5F7F-4211-9042-42D8BCE60415}.Release|Any CPU.ActiveCfg = Release|Any CPU {6F591D05-5F7F-4211-9042-42D8BCE60415}.Release|Any CPU.Build.0 = Release|Any CPU - {232E1153-6366-4175-A982-D66B30AAD610}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {232E1153-6366-4175-A982-D66B30AAD610}.Debug|Any CPU.Build.0 = Debug|Any CPU - {232E1153-6366-4175-A982-D66B30AAD610}.Publish|Any CPU.ActiveCfg = Debug|Any CPU - {232E1153-6366-4175-A982-D66B30AAD610}.Publish|Any CPU.Build.0 = Debug|Any CPU - {232E1153-6366-4175-A982-D66B30AAD610}.Release|Any CPU.ActiveCfg = Release|Any CPU - {232E1153-6366-4175-A982-D66B30AAD610}.Release|Any CPU.Build.0 = Release|Any CPU {E82B640C-1704-430D-8D71-FD8ED3695468}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {E82B640C-1704-430D-8D71-FD8ED3695468}.Debug|Any CPU.Build.0 = Debug|Any CPU {E82B640C-1704-430D-8D71-FD8ED3695468}.Publish|Any CPU.ActiveCfg = Debug|Any CPU @@ -1126,6 +1122,12 @@ Global {39EAB599-742F-417D-AF80-95F90376BB18}.Publish|Any CPU.Build.0 = Publish|Any CPU {39EAB599-742F-417D-AF80-95F90376BB18}.Release|Any CPU.ActiveCfg = Release|Any CPU {39EAB599-742F-417D-AF80-95F90376BB18}.Release|Any CPU.Build.0 = Release|Any CPU + {232E1153-6366-4175-A982-D66B30AAD610}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {232E1153-6366-4175-A982-D66B30AAD610}.Debug|Any CPU.Build.0 = Debug|Any CPU + {232E1153-6366-4175-A982-D66B30AAD610}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {232E1153-6366-4175-A982-D66B30AAD610}.Publish|Any CPU.Build.0 = Debug|Any CPU + {232E1153-6366-4175-A982-D66B30AAD610}.Release|Any CPU.ActiveCfg = Release|Any CPU + {232E1153-6366-4175-A982-D66B30AAD610}.Release|Any CPU.Build.0 = Release|Any CPU {DAC54048-A39A-4739-8307-EA5A291F2EA0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {DAC54048-A39A-4739-8307-EA5A291F2EA0}.Debug|Any CPU.Build.0 = Debug|Any CPU {DAC54048-A39A-4739-8307-EA5A291F2EA0}.Publish|Any CPU.ActiveCfg = Debug|Any CPU @@ -1264,6 +1266,12 @@ Global {06181F0F-A375-43AE-B45F-73CBCFC30C14}.Publish|Any CPU.Build.0 = Debug|Any CPU {06181F0F-A375-43AE-B45F-73CBCFC30C14}.Release|Any CPU.ActiveCfg = Release|Any CPU {06181F0F-A375-43AE-B45F-73CBCFC30C14}.Release|Any CPU.Build.0 = Release|Any CPU + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Publish|Any CPU.ActiveCfg = Debug|Any CPU + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Publish|Any CPU.Build.0 = Debug|Any CPU + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -1437,6 +1445,7 @@ Global {A0E65043-6B00-4836-850F-000A52238914} = {4F381919-F1BE-47D8-8558-3187ED04A84F} {11DFBF14-6FBA-41F0-B7F3-A288952D6FDB} = {4F381919-F1BE-47D8-8558-3187ED04A84F} {06181F0F-A375-43AE-B45F-73CBCFC30C14} = {4F381919-F1BE-47D8-8558-3187ED04A84F} + {9D37CD08-620D-4AAC-9FEC-A8126AD8AB56} = {4F381919-F1BE-47D8-8558-3187ED04A84F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj new file mode 100644 index 000000000000..82ddc0f3446c --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj @@ -0,0 +1,49 @@ + + + + net8.0;net472 + enable + enable + + false + true + + $(NoWarn);CA2007,SKEXP0001,SKEXP0020,VSTHRD111;CS1685 + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 + + + + + + runtime; build; native; contentfiles; analyzers; buildtransitive + all + + + + + + + + + + + + + + + + + + + + + + + Always + + + Always + + + + \ No newline at end of file diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs new file mode 100644 index 000000000000..9f316131e21e --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Pinecone; +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.Xunit; +using Xunit; + +namespace PineconeIntegrationTests; + +public class SampleTests(PineconeFixture fixture) : IClassFixture +{ + [ConditionalFact] + public async Task CanRunSampleCode() + { + var collectionModel = await fixture.Client.CreateCollectionAsync(new CreateCollectionRequest + { + Name = "example-collection", + Source = "example-index", + }); + + await fixture.Client.DeleteCollectionAsync(collectionModel.Name); + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs new file mode 100644 index 000000000000..21bbdfd51eaf --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeFixture.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Pinecone; +using VectorDataSpecificationTests.Support; + +namespace PineconeIntegrationTests.Support; + +public class PineconeFixture : VectorStoreFixture +{ + public override TestStore TestStore => PineconeTestStore.Instance; + + public PineconeClient Client => PineconeTestStore.Instance.Client; +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs new file mode 100644 index 000000000000..5770ef9379fd --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. + +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Containers; +using Microsoft.Extensions.VectorData; +using Pinecone; +using VectorDataSpecificationTests.Support; + +namespace PineconeIntegrationTests.Support; + +#pragma warning disable CA1001 // Type owns disposable fields but is not disposable + +internal sealed class PineconeTestStore : TestStore +{ + // Values taken from https://docs.pinecone.io/guides/operations/local-development + private const string Image = "ghcr.io/pinecone-io/pinecone-local:v0.7.0"; + private const ushort HttpPort = 5080; + private const string Host = "localhost"; + + public static PineconeTestStore Instance { get; } = new(); + + private IContainer? _container; + private PineconeClient? _client; + private IVectorStore? _defaultVectorStore; + + public PineconeClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); + + public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized"); + + private PineconeTestStore() + { + } + + protected override async Task StartAsync() + { + this._container = await this.CreateContainerAsync(); + + string url = $"http://{this._container.Hostname}:{this._container.GetMappedPublicPort(HttpPort)}"; + + this._client = new PineconeClient( + apiKey: "ForPineconeLocalTheApiKeysAreIgnored", + clientOptions: new() + { + BaseUrl = url, + MaxRetries = 0, + IsTlsEnabled = false + }); + //this._defaultVectorStore = new(this._client); + + Console.WriteLine(url); + } + + protected override async Task StopAsync() + { + if (this._container is not null) + { + await this._container.DisposeAsync(); + } + } + + private async Task CreateContainerAsync() + { + var container = new ContainerBuilder() + .WithImage(Image) + .WithPortBinding(HttpPort, assignRandomHostPort: true) + .WithEnvironment("PINECONE_HOST", "localhost") + .WithEnvironment("PORT", HttpPort.ToString()) + .WithHostname(Host) + .WithExposedPort(HttpPort) + //.WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(r => r.ForPort(HttpPort))) + //.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(HttpPort)) + //.WithWaitStrategy(Wait.ForUnixContainer().UntilMessageIsLogged(".*Emulating a Pinecone serverless index.*")) + .Build(); + + await container.StartAsync(); + + return container; + } +} From 9f03e45451ef68955451d0d44e2b9c8579645b5f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 3 Mar 2025 20:21:05 +0100 Subject: [PATCH 2/8] get the test to work --- dotnet/Directory.Packages.props | 2 +- .../PineconeIntegrationTests/SampleTests.cs | 20 +++++++-- .../Support/PineconeTestStore.cs | 44 +++++++++++-------- 3 files changed, 42 insertions(+), 24 deletions(-) diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 21ff6dba6394..ce4f39f93f15 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -46,7 +46,7 @@ - + diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs index 9f316131e21e..89138368cc57 100644 --- a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs @@ -12,12 +12,24 @@ public class SampleTests(PineconeFixture fixture) : IClassFixture CreateContainerAsync() { var container = new ContainerBuilder() .WithImage(Image) - .WithPortBinding(HttpPort, assignRandomHostPort: true) - .WithEnvironment("PINECONE_HOST", "localhost") - .WithEnvironment("PORT", HttpPort.ToString()) - .WithHostname(Host) - .WithExposedPort(HttpPort) - //.WithWaitStrategy(Wait.ForUnixContainer().UntilHttpRequestIsSucceeded(r => r.ForPort(HttpPort))) - //.WithWaitStrategy(Wait.ForUnixContainer().UntilPortIsAvailable(HttpPort)) - //.WithWaitStrategy(Wait.ForUnixContainer().UntilMessageIsLogged(".*Emulating a Pinecone serverless index.*")) + .WithPortBinding(RestPort, assignRandomHostPort: true) + .WithPortBinding(GrpcPort, assignRandomHostPort: true) .Build(); await container.StartAsync(); From acd2feb67cb3dbca339e56f3a27619d1705380f7 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 3 Mar 2025 20:42:24 +0100 Subject: [PATCH 3/8] port ListCollectionsAsync, CollectionExistsAsync, CreateCollectionAsync and DeleteCollectionAsync and Upsert*Async --- .../Connectors.Memory.Pinecone.csproj | 2 +- .../PineconeVectorStore.cs | 17 +- ...econeVectorStoreCollectionCreateMapping.cs | 46 --- .../PineconeVectorStoreRecordCollection.cs | 279 ++++++++++-------- .../PineconeIntegrationTests.csproj | 2 +- .../Support/PineconeTestStore.cs | 11 +- 6 files changed, 165 insertions(+), 192 deletions(-) delete mode 100644 dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj index b2127f5131b0..3b51b03623ae 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj @@ -19,7 +19,7 @@ - + diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs index 4f79810e641b..3dc2da7eca6a 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading; -using Grpc.Core; using Microsoft.Extensions.VectorData; using Pinecone; using Sdk = Pinecone; @@ -20,7 +19,6 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; public class PineconeVectorStore : IVectorStore { private const string DatabaseName = "Pinecone"; - private const string ListCollectionsName = "ListCollections"; private readonly Sdk.PineconeClient _pineconeClient; private readonly PineconeVectorStoreOptions _options; @@ -63,24 +61,27 @@ public virtual IVectorStoreRecordCollection GetCollection public virtual async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) { - IndexDetails[] collections; + IndexList indexList; try { - collections = await this._pineconeClient.ListIndexes(cancellationToken).ConfigureAwait(false); + indexList = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - catch (RpcException ex) + catch (Exception ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { VectorStoreType = DatabaseName, - OperationName = ListCollectionsName + OperationName = "ListCollections" }; } - foreach (var collection in collections) + if (indexList.Indexes is not null) { - yield return collection.Name; + foreach (var index in indexList.Indexes) + { + yield return index.Name; + } } } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs deleted file mode 100644 index 5f8d6bf6137d..000000000000 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionCreateMapping.cs +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData; -using Pinecone; - -namespace Microsoft.SemanticKernel.Connectors.Pinecone; - -/// -/// Contains mapping helpers to use when creating a Pinecone vector collection. -/// -internal static class PineconeVectorStoreCollectionCreateMapping -{ - /// - /// Maps information stored in to a structure used by Pinecone SDK to create a serverless index. - /// - /// The property to map. - /// The structure containing settings used to create a serverless index. - /// Thrown if the property is missing information or has unsupported options specified. - public static (uint Dimension, Metric Metric) MapServerlessIndex(VectorStoreRecordVectorProperty vectorProperty) - { - if (vectorProperty!.Dimensions is not > 0) - { - throw new InvalidOperationException($"Property {nameof(vectorProperty.Dimensions)} on {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' must be set to a positive integer to create a collection."); - } - - return (Dimension: (uint)vectorProperty.Dimensions, Metric: GetSDKMetricAlgorithm(vectorProperty)); - } - - /// - /// Get the configured from the given . - /// If none is configured, the default is . - /// - /// The vector property definition. - /// The chosen . - /// Thrown if a distance function is chosen that isn't supported by Pinecone. - public static Metric GetSDKMetricAlgorithm(VectorStoreRecordVectorProperty vectorProperty) - => vectorProperty.DistanceFunction switch - { - DistanceFunction.CosineSimilarity => Metric.Cosine, - DistanceFunction.DotProductSimilarity => Metric.DotProduct, - DistanceFunction.EuclideanSquaredDistance => Metric.Euclidean, - null => Metric.Cosine, - _ => throw new InvalidOperationException($"Distance function '{vectorProperty.DistanceFunction}' for {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' is not supported by the Pinecone VectorStore.") - }; -} diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 1db3d86fbf16..4df993f3d0df 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -9,7 +9,6 @@ using Grpc.Core; using Microsoft.Extensions.VectorData; using Pinecone; -using Pinecone.Grpc; using Sdk = Pinecone; namespace Microsoft.SemanticKernel.Connectors.Pinecone; @@ -23,14 +22,6 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo #pragma warning restore CA1711 // Identifiers should not have incorrect suffix { private const string DatabaseName = "Pinecone"; - private const string CreateCollectionName = "CreateCollection"; - private const string CollectionExistsName = "CollectionExists"; - private const string DeleteCollectionName = "DeleteCollection"; - - private const string UpsertOperationName = "Upsert"; - private const string DeleteOperationName = "Delete"; - private const string GetOperationName = "Get"; - private const string QueryOperationName = "Query"; private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new(); @@ -38,8 +29,7 @@ public class PineconeVectorStoreRecordCollection : IVectorStoreRecordCo private readonly PineconeVectorStoreRecordCollectionOptions _options; private readonly VectorStoreRecordPropertyReader _propertyReader; private readonly IVectorStoreRecordMapper _mapper; - - private Sdk.Index? _index; + private IndexClient? _indexClient; /// public string CollectionName { get; } @@ -90,36 +80,39 @@ public PineconeVectorStoreRecordCollection(Sdk.PineconeClient pineconeClient, st } /// - public virtual async Task CollectionExistsAsync(CancellationToken cancellationToken = default) - { - var result = await this.RunOperationAsync( - CollectionExistsName, + public virtual Task CollectionExistsAsync(CancellationToken cancellationToken = default) + => this.RunOperationAsync( + "CollectionExists", async () => { - var collections = await this._pineconeClient.ListIndexes(cancellationToken).ConfigureAwait(false); + var collections = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - return collections.Any(x => x.Name == this.CollectionName); - }).ConfigureAwait(false); - - return result; - } + return collections.Indexes?.Any(x => x.Name == this.CollectionName) is true; + }); /// - public virtual async Task CreateCollectionAsync(CancellationToken cancellationToken = default) + public virtual Task CreateCollectionAsync(CancellationToken cancellationToken = default) { // we already run through record property validation, so a single VectorStoreRecordVectorProperty is guaranteed. var vectorProperty = this._propertyReader.VectorProperty!; - var (dimension, metric) = PineconeVectorStoreCollectionCreateMapping.MapServerlessIndex(vectorProperty); - await this.RunOperationAsync( - CreateCollectionName, - () => this._pineconeClient.CreateServerlessIndex( - this.CollectionName, - dimension, - metric, - this._options.ServerlessIndexCloud, - this._options.ServerlessIndexRegion, - cancellationToken)).ConfigureAwait(false); + CreateIndexRequest request = new() + { + Name = this.CollectionName, + Dimension = vectorProperty.Dimensions ?? throw new InvalidOperationException($"Property {nameof(vectorProperty.Dimensions)} on {nameof(VectorStoreRecordVectorProperty)} '{vectorProperty.DataModelPropertyName}' must be set to a positive integer to create a collection."), + Metric = MapDistanceFunction(vectorProperty), + Spec = new ServerlessIndexSpec + { + Serverless = new ServerlessSpec + { + Cloud = MapCloud(this._options.ServerlessIndexCloud), + Region = this._options.ServerlessIndexRegion, + } + }, + }; + + return this.RunOperationAsync("CreateCollection", + () => this._pineconeClient.CreateIndexAsync(request, cancellationToken: cancellationToken)); } /// @@ -127,24 +120,52 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can { if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) { - await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + try + { + await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + } + catch (PineconeApiException ex) when (ex.InnerException is PineconeApiException) + { + // If the collection already exists, we should ignore the exception. + // TODO adsitnik: find out which exception is thrown when the collection already exists. + throw; + } } } /// public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = default) => this.RunOperationAsync( - DeleteCollectionName, - () => this._pineconeClient.DeleteIndex(this.CollectionName, cancellationToken)); + "DeleteCollection", + () => this._pineconeClient.DeleteIndexAsync(this.CollectionName, cancellationToken: cancellationToken)); /// public virtual async Task GetAsync(string key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) { Verify.NotNull(key); - var records = await this.GetBatchAsync([key], options, cancellationToken).ToListAsync(cancellationToken).ConfigureAwait(false); + Sdk.FetchRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = [key] + }; + + var response = await this.RunOperationAsync( + "Get", + () => this.GetIndexClient().FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); - return records.FirstOrDefault(); + var result = response.Vectors?.Values.FirstOrDefault(); + if (result is null) + { + return default; + } + + StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true }; + return VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + "Get", + () => this._mapper.MapFromStorageToDataModel(result, mapperOptions)); } /// @@ -155,20 +176,32 @@ public virtual async IAsyncEnumerable GetBatchAsync( { Verify.NotNull(keys); - var indexNamespace = this.GetIndexNamespace(); - var mapperOptions = new StorageToDataModelMapperOptions { IncludeVectors = options?.IncludeVectors ?? false }; - - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); + List keysList = keys.ToList(); + if (keysList.Count == 0) + { + yield break; + } - var results = await this.RunOperationAsync( - GetOperationName, - () => index.Fetch(keys, indexNamespace, cancellationToken)).ConfigureAwait(false); + Sdk.FetchRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = keysList + }; + + var response = await this.RunOperationAsync( + "GetBatch", + () => this.GetIndexClient().FetchAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + if (response.Vectors is null || response.Vectors.Count == 0) + { + yield break; + } + StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options?.IncludeVectors is true }; var records = VectorStoreErrorHandler.RunModelConversion( DatabaseName, this.CollectionName, - GetOperationName, - () => results.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); + "GetBatch", + () => response.Vectors.Values.Select(x => this._mapper.MapFromStorageToDataModel(x, mapperOptions))); foreach (var record in records) { @@ -181,21 +214,37 @@ public virtual Task DeleteAsync(string key, CancellationToken cancellationToken { Verify.NotNullOrWhiteSpace(key); - return this.DeleteBatchAsync([key], cancellationToken); + Sdk.DeleteRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = [key] + }; + + return this.RunOperationAsync( + "Delete", + () => this.GetIndexClient().DeleteAsync(request, cancellationToken: cancellationToken)); } /// - public virtual async Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default) + public virtual Task DeleteBatchAsync(IEnumerable keys, CancellationToken cancellationToken = default) { Verify.NotNull(keys); - var indexNamespace = this.GetIndexNamespace(); + List keysList = keys.ToList(); + if (keysList.Count == 0) + { + return Task.CompletedTask; + } - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); + Sdk.DeleteRequest request = new() + { + Namespace = this._options.IndexNamespace, + Ids = keysList + }; - await this.RunOperationAsync( - DeleteOperationName, - () => index.Delete(keys, indexNamespace, cancellationToken)).ConfigureAwait(false); + return this.RunOperationAsync( + "DeleteBatch", + () => this.GetIndexClient().DeleteAsync(request, cancellationToken: cancellationToken)); } /// @@ -203,19 +252,21 @@ public virtual async Task UpsertAsync(TRecord record, CancellationToken { Verify.NotNull(record); - var indexNamespace = this.GetIndexNamespace(); - - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); - var vector = VectorStoreErrorHandler.RunModelConversion( DatabaseName, this.CollectionName, - UpsertOperationName, + "Upsert", () => this._mapper.MapFromDataToStorageModel(record)); + Sdk.UpsertRequest request = new() + { + Namespace = this._options.IndexNamespace, + Vectors = [vector], + }; + await this.RunOperationAsync( - UpsertOperationName, - () => index.Upsert([vector], indexNamespace, cancellationToken)).ConfigureAwait(false); + "Upsert", + () => this.GetIndexClient().UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); return vector.Id; } @@ -225,19 +276,27 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable records.Select(this._mapper.MapFromDataToStorageModel).ToList()); + if (vectors.Count == 0) + { + yield break; + } + + Sdk.UpsertRequest request = new() + { + Namespace = this._options.IndexNamespace, + Vectors = vectors, + }; + + var indexClient = this._pineconeClient.Index(name: this.CollectionName); await this.RunOperationAsync( - UpsertOperationName, - () => index.Upsert(vectors, indexNamespace, cancellationToken)).ConfigureAwait(false); + "UpsertBatch", + () => indexClient.UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); foreach (var vector in vectors) { @@ -256,61 +315,7 @@ public virtual async Task> VectorizedSearchAsync).FullName}"); } - // Resolve options and build filter clause. - var internalOptions = options ?? s_defaultVectorSearchOptions; - var mapperOptions = new StorageToDataModelMapperOptions { IncludeVectors = options?.IncludeVectors ?? false }; - -#pragma warning disable CS0618 // FilterClause is obsolete - var filter = PineconeVectorStoreCollectionSearchMapping.BuildSearchFilter( - internalOptions.Filter?.FilterClauses, - this._propertyReader.StoragePropertyNamesMap); -#pragma warning restore CS0618 - - // Get the current index. - var indexNamespace = this.GetIndexNamespace(); - var index = await this.GetIndexAsync(this.CollectionName, cancellationToken).ConfigureAwait(false); - - // Search. - var results = await this.RunOperationAsync( - QueryOperationName, - () => index.Query( - floatVector.ToArray(), - (uint)(internalOptions.Skip + internalOptions.Top), - filter, - sparseValues: null, - indexNamespace, - internalOptions.IncludeVectors, - includeMetadata: true, - cancellationToken)).ConfigureAwait(false); - - // Skip the required results for paging. - var skippedResults = results.Skip(internalOptions.Skip); - - // Map the results. - var records = VectorStoreErrorHandler.RunModelConversion( - DatabaseName, - this.CollectionName, - QueryOperationName, - () => - { - // First convert to Vector objects, since the - // mapper requires these as input. - var vectorResults = skippedResults.Select(x => ( - Vector: new Vector() - { - Id = x.Id, - Values = x.Values ?? Array.Empty(), - Metadata = x.Metadata, - SparseValues = x.SparseValues - }, - x.Score)); - - return vectorResults.Select(x => new VectorSearchResult( - this._mapper.MapFromStorageToDataModel(x.Vector, mapperOptions), - x.Score)); - }); - - return new VectorSearchResults(records.ToAsyncEnumerable()); + throw new NotImplementedException(); } private async Task RunOperationAsync(string operationName, Func> operation) @@ -336,7 +341,7 @@ private async Task RunOperationAsync(string operationName, Func operation) { await operation.Invoke().ConfigureAwait(false); } - catch (RpcException ex) + catch (PineconeApiException ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { @@ -347,13 +352,25 @@ private async Task RunOperationAsync(string operationName, Func operation) } } - private async Task> GetIndexAsync(string indexName, CancellationToken cancellationToken) - { - this._index ??= await this._pineconeClient.GetIndex(indexName, cancellationToken).ConfigureAwait(false); - - return this._index; - } + private IndexClient GetIndexClient() + => this._indexClient ??= this._pineconeClient.Index(name: this.CollectionName); - private string? GetIndexNamespace() - => this._options.IndexNamespace; + private static ServerlessSpecCloud MapCloud(string serverlessIndexCloud) + => serverlessIndexCloud switch + { + "aws" => ServerlessSpecCloud.Aws, + "azure" => ServerlessSpecCloud.Azure, + "gcp" => ServerlessSpecCloud.Gcp, + _ => throw new ArgumentException($"Invalid serverless index cloud: {serverlessIndexCloud}.", nameof(serverlessIndexCloud)) + }; + + private static CreateIndexRequestMetric MapDistanceFunction(VectorStoreRecordVectorProperty vectorProperty) + => vectorProperty.DistanceFunction switch + { + DistanceFunction.CosineSimilarity => CreateIndexRequestMetric.Cosine, + DistanceFunction.DotProductSimilarity => CreateIndexRequestMetric.Dotproduct, + DistanceFunction.EuclideanSquaredDistance => CreateIndexRequestMetric.Euclidean, + null => CreateIndexRequestMetric.Cosine, + _ => throw new NotSupportedException($"Distance function '{vectorProperty.DistanceFunction}' is not supported.") + }; } diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj index 82ddc0f3446c..bc70e11b12dd 100644 --- a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/PineconeIntegrationTests.csproj @@ -33,7 +33,7 @@ - + diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs index 82418703e00f..bfedfe549d17 100644 --- a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs @@ -4,6 +4,7 @@ using DotNet.Testcontainers.Containers; using Grpc.Net.Client; using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.Pinecone; using Pinecone; using VectorDataSpecificationTests.Support; @@ -21,10 +22,10 @@ internal sealed class PineconeTestStore : TestStore public static PineconeTestStore Instance { get; } = new(); private IContainer? _container; - private PineconeClient? _client; - private IVectorStore? _defaultVectorStore; + private Pinecone.PineconeClient? _client; + private PineconeVectorStore? _defaultVectorStore; - public PineconeClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); + public Pinecone.PineconeClient Client => this._client ?? throw new InvalidOperationException("Not initialized"); public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized"); @@ -55,11 +56,11 @@ protected override async Task StartAsync() GrpcOptions = grpcOptions }; - this._client = new PineconeClient( + this._client = new Pinecone.PineconeClient( apiKey: "ForPineconeLocalTheApiKeysAreIgnored", clientOptions: clientOptions); - //this._defaultVectorStore = new(this._client); + this._defaultVectorStore = new(this._client); } protected override async Task StopAsync() From 4a37c6002e7141d479ed27a0d168afed4f7a591e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 4 Mar 2025 10:43:29 +0100 Subject: [PATCH 4/8] port the mappers and search filter logic --- .../PineconeGenericDataModelMapper.cs | 17 +++--- ...econeVectorStoreCollectionSearchMapping.cs | 5 +- .../PineconeVectorStoreRecordCollection.cs | 52 +++++++++++++++++-- .../PineconeVectorStoreRecordFieldMapping.cs | 20 +++---- .../PineconeVectorStoreRecordMapper.cs | 6 +-- 5 files changed, 74 insertions(+), 26 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs index 496a848ed394..98ecab5c1a80 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs @@ -34,7 +34,7 @@ public PineconeGenericDataModelMapper( /// public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel dataModel) { - var metadata = new MetadataMap(); + var metadata = new Metadata(); // Map data properties. foreach (var dataProperty in this._propertyReader.DataProperties) @@ -42,9 +42,10 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data if (dataModel.Data.TryGetValue(dataProperty.DataModelPropertyName, out var propertyValue)) { var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName); - metadata[propertyStorageName] = propertyValue == null ? - new MetadataValue() : - PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue); + if (propertyValue is not null) + { + metadata[propertyStorageName] = PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue); + } } } @@ -62,8 +63,8 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data // TODO: what about sparse values? var result = new Vector { - Id = (string)dataModel.Key, - Values = values.ToArray(), + Id = dataModel.Key, + Values = values, Metadata = metadata, SparseValues = null }; @@ -80,7 +81,7 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor // Set Vector. if (options?.IncludeVectors is true) { - dataModel.Vectors.Add(this._propertyReader.FirstVectorPropertyName!, new ReadOnlyMemory(storageModel.Values)); + dataModel.Vectors.Add(this._propertyReader.FirstVectorPropertyName!, storageModel.Values); } // Set Data. @@ -89,7 +90,7 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor foreach (var dataProperty in this._propertyReader.DataProperties) { var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName); - if (storageModel.Metadata.TryGetValue(propertyStorageName, out var propertyValue)) + if (storageModel.Metadata.TryGetValue(propertyStorageName, out var propertyValue) && propertyValue is not null) { dataModel.Data[dataProperty.DataModelPropertyName] = PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType( propertyValue, diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs index 5b3d511c6b08..89d66429313f 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs @@ -20,9 +20,9 @@ internal static class PineconeVectorStoreCollectionSearchMapping /// A mapping from property name to the name under which the property would be stored. /// The Pinecone . /// Thrown for invalid property names, value types or filter clause types. - public static MetadataMap BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap) + public static Metadata BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap) { - var metadataMap = new MetadataMap(); + var metadataMap = new Metadata(); if (filterClauses is null) { @@ -46,7 +46,6 @@ public static MetadataMap BuildSearchFilter(IEnumerable? filterCla bool boolValue => (MetadataValue)boolValue, float floatValue => (MetadataValue)floatValue, double doubleValue => (MetadataValue)doubleValue, - decimal decimalValue => (MetadataValue)decimalValue, _ => throw new InvalidOperationException($"Unsupported filter value type '{equalToFilterClause.Value.GetType().Name}'.") }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 4df993f3d0df..ca5c9c81ff49 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -293,10 +293,9 @@ public virtual async IAsyncEnumerable UpsertBatchAsync(IEnumerable indexClient.UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + () => this.GetIndexClient().UpsertAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); foreach (var vector in vectors) { @@ -315,7 +314,54 @@ public virtual async Task> VectorizedSearchAsync).FullName}"); } - throw new NotImplementedException(); + options ??= s_defaultVectorSearchOptions; + +#pragma warning disable CS0618 // FilterClause is obsolete + var filter = PineconeVectorStoreCollectionSearchMapping.BuildSearchFilter( + options.Filter?.FilterClauses, + this._propertyReader.StoragePropertyNamesMap); +#pragma warning restore CS0618 + + Sdk.QueryRequest request = new() + { + TopK = (uint)(options.Top + options.Skip), + Namespace = this._options.IndexNamespace, + IncludeValues = options.IncludeVectors, + IncludeMetadata = true, + Vector = floatVector, + Filter = filter, + }; + + Sdk.QueryResponse response = await this.RunOperationAsync( + "Query", + () => this.GetIndexClient().QueryAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); + + if (response.Results is null) + { + return new VectorSearchResults(Array.Empty>().ToAsyncEnumerable()); + } + + // Pinecone does not provide a way to skip results, so we need to do it manually. + var skippedResults = response.Results + .Where(result => result.Matches is not null) + .SelectMany(result => result.Matches!) + .Skip(options.Skip); + + StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options.IncludeVectors is true }; + var records = VectorStoreErrorHandler.RunModelConversion( + DatabaseName, + this.CollectionName, + "Query", + () => skippedResults.Select(x => new VectorSearchResult(this._mapper.MapFromStorageToDataModel(new Sdk.Vector() + { + Id = x.Id, + Values = x.Values ?? Array.Empty(), + Metadata = x.Metadata, + SparseValues = x.SparseValues + }, mapperOptions), x.Score))) + .ToAsyncEnumerable(); + + return new(records); } private async Task RunOperationAsync(string operationName, Func> operation) diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs index 6acbec24d72c..9573740f8580 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordFieldMapping.cs @@ -48,7 +48,7 @@ internal static class PineconeVectorStoreRecordFieldMapping ]; public static object? ConvertFromMetadataValueToNativeType(MetadataValue metadataValue, Type targetType) - => metadataValue.Inner switch + => metadataValue.Value switch { null => null, bool boolValue => boolValue, @@ -59,26 +59,28 @@ internal static class PineconeVectorStoreRecordFieldMapping long longValue => ConvertToNumericValue(longValue, targetType), float floatValue => ConvertToNumericValue(floatValue, targetType), double doubleValue => ConvertToNumericValue(doubleValue, targetType), - decimal decimalValue => ConvertToNumericValue(decimalValue, targetType), MetadataValue[] array => VectorStoreRecordMapping.CreateEnumerable(array.Select(x => ConvertFromMetadataValueToNativeType(x, VectorStoreRecordPropertyVerification.GetCollectionElementType(targetType))), targetType), List list => VectorStoreRecordMapping.CreateEnumerable(list.Select(x => ConvertFromMetadataValueToNativeType(x, VectorStoreRecordPropertyVerification.GetCollectionElementType(targetType))), targetType), - _ => throw new VectorStoreRecordMappingException($"Unsupported metadata type: '{metadataValue.Inner?.GetType().FullName}'."), + _ => throw new VectorStoreRecordMappingException($"Unsupported metadata type: '{metadataValue.Value?.GetType().FullName}'."), }; - // TODO: take advantage of MetadataValue.TryCreate once we upgrade the version of Pinecone.NET public static MetadataValue ConvertToMetadataValue(object? sourceValue) => sourceValue switch { bool boolValue => boolValue, + bool[] bools => bools, + List bools => bools, string stringValue => stringValue, + string[] stringArray => stringArray, + List stringList => stringList, + double doubleValue => doubleValue, + double[] doubles => doubles, + List doubles => doubles, + // Other numeric types are simply cast into double in implicit way. + // We could consider supporting arrays of these types. int intValue => intValue, long longValue => longValue, float floatValue => floatValue, - double doubleValue => doubleValue, - decimal decimalValue => decimalValue, - string[] stringArray => stringArray, - List stringList => stringList, - IEnumerable stringEnumerable => stringEnumerable.ToArray(), _ => throw new VectorStoreRecordMappingException($"Unsupported source value type '{sourceValue?.GetType().FullName}'.") }; diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs index 501937eaf50d..500bf98e2b87 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs @@ -40,7 +40,7 @@ public Vector MapFromDataToStorageModel(TRecord dataModel) throw new VectorStoreRecordMappingException($"Key property {this._propertyReader.KeyPropertyName} on provided record of type {typeof(TRecord).FullName} may not be null."); } - var metadata = new MetadataMap(); + var metadata = new Metadata(); foreach (var dataPropertyInfo in this._propertyReader.DataPropertiesInfo) { var propertyName = this._propertyReader.GetStoragePropertyName(dataPropertyInfo.Name); @@ -61,7 +61,7 @@ public Vector MapFromDataToStorageModel(TRecord dataModel) var result = new Vector { Id = (string)keyObject, - Values = values.ToArray(), + Values = values, Metadata = metadata, SparseValues = null }; @@ -83,7 +83,7 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, StorageToDataModel { this._propertyReader.FirstVectorPropertyInfo!.SetValue( outputRecord, - new ReadOnlyMemory(storageModel.Values)); + storageModel.Values); } // Set Data. From cef33580500b4beb35654c77598904498df14195 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 4 Mar 2025 17:27:31 +0100 Subject: [PATCH 5/8] polishing after reading the code again --- .../PineconeVectorStore.cs | 2 +- .../PineconeVectorStoreCollectionSearchMapping.cs | 6 +++--- .../PineconeVectorStoreRecordCollection.cs | 13 +++---------- .../PineconeVectorStoreRecordMapper.cs | 2 +- 4 files changed, 8 insertions(+), 15 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs index 3dc2da7eca6a..a072ea6e7336 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStore.cs @@ -67,7 +67,7 @@ public virtual async IAsyncEnumerable ListCollectionNamesAsync([Enumerat { indexList = await this._pineconeClient.ListIndexesAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - catch (Exception ex) + catch (PineconeApiException ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs index 89d66429313f..f5106bb251c1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreCollectionSearchMapping.cs @@ -14,11 +14,11 @@ internal static class PineconeVectorStoreCollectionSearchMapping { #pragma warning disable CS0618 // FilterClause is obsolete /// - /// Build a Pinecone from a set of filter clauses. + /// Build a Pinecone from a set of filter clauses. /// - /// The filter clauses to build the Pinecone from. + /// The filter clauses to build the Pinecone from. /// A mapping from property name to the name under which the property would be stored. - /// The Pinecone . + /// The Pinecone . /// Thrown for invalid property names, value types or filter clause types. public static Metadata BuildSearchFilter(IEnumerable? filterClauses, IReadOnlyDictionary storagePropertyNamesMap) { diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index ca5c9c81ff49..74692db44a15 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -120,16 +120,9 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can { if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) { - try - { - await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); - } - catch (PineconeApiException ex) when (ex.InnerException is PineconeApiException) - { - // If the collection already exists, we should ignore the exception. - // TODO adsitnik: find out which exception is thrown when the collection already exists. - throw; - } + await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + // If the collection already exists, we should ignore the exception. + // TODO adsitnik: find out which exception is thrown when the collection already exists. } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs index 500bf98e2b87..1163c1a66bea 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordMapper.cs @@ -94,7 +94,7 @@ public TRecord MapFromStorageToDataModel(Vector storageModel, StorageToDataModel this._propertyReader.DataPropertiesInfo, this._propertyReader.StoragePropertyNamesMap, storageModel.Metadata, - PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType); + PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType!); } return outputRecord; From 9cd429c5ff993b4713d57a6db8b65e3a2c6996ce Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 10 Mar 2025 11:13:40 +0100 Subject: [PATCH 6/8] port the unit test project, make sure all of them pass --- .../PineconeGenericDataModelMapper.cs | 16 +++---- .../PineconeGenericDataModelMapperTests.cs | 48 +++++++++---------- .../PineconeKernelBuilderExtensionsTests.cs | 4 +- ...ineconeServiceCollectionExtensionsTests.cs | 4 +- ...ineconeVectorStoreRecordCollectionTests.cs | 2 +- 5 files changed, 37 insertions(+), 37 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs index 98ecab5c1a80..df783a230498 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeGenericDataModelMapper.cs @@ -42,10 +42,9 @@ public Vector MapFromDataToStorageModel(VectorStoreGenericDataModel data if (dataModel.Data.TryGetValue(dataProperty.DataModelPropertyName, out var propertyValue)) { var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName); - if (propertyValue is not null) - { - metadata[propertyStorageName] = PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue); - } + metadata[propertyStorageName] = propertyValue is not null + ? PineconeVectorStoreRecordFieldMapping.ConvertToMetadataValue(propertyValue) + : null; } } @@ -90,11 +89,12 @@ public VectorStoreGenericDataModel MapFromStorageToDataModel(Vector stor foreach (var dataProperty in this._propertyReader.DataProperties) { var propertyStorageName = this._propertyReader.GetStoragePropertyName(dataProperty.DataModelPropertyName); - if (storageModel.Metadata.TryGetValue(propertyStorageName, out var propertyValue) && propertyValue is not null) + if (storageModel.Metadata.TryGetValue(propertyStorageName, out var propertyValue)) { - dataModel.Data[dataProperty.DataModelPropertyName] = PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType( - propertyValue, - dataProperty.PropertyType); + dataModel.Data[dataProperty.DataModelPropertyName] = + propertyValue is not null + ? PineconeVectorStoreRecordFieldMapping.ConvertFromMetadataValueToNativeType(propertyValue, dataProperty.PropertyType) + : null; } } } diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs index 9a41450d9649..0a96bb41cc3b 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeGenericDataModelMapperTests.cs @@ -77,20 +77,20 @@ public void MapFromDataToStorageModelMapsAllSupportedTypes() // Assert Assert.Equal(TestKeyString, storageModel.Id); - Assert.Equal("string", (string?)storageModel.Metadata!["StringDataProp"].Inner); + Assert.Equal("string", (string?)storageModel.Metadata!["StringDataProp"]!.Value); // MetadataValue converts all numeric types to double. - Assert.Equal(1, (double?)storageModel.Metadata["IntDataProp"].Inner); - Assert.Equal(2, (double?)storageModel.Metadata["NullableIntDataProp"].Inner); - Assert.Equal(3L, (double?)storageModel.Metadata["LongDataProp"].Inner); - Assert.Equal(4L, (double?)storageModel.Metadata["NullableLongDataProp"].Inner); - Assert.Equal(5.0f, (double?)storageModel.Metadata["FloatDataProp"].Inner); - Assert.Equal(6.0f, (double?)storageModel.Metadata["NullableFloatDataProp"].Inner); - Assert.Equal(7.0, (double?)storageModel.Metadata["DoubleDataProp"].Inner); - Assert.Equal(8.0, (double?)storageModel.Metadata["NullableDoubleDataProp"].Inner); - Assert.Equal(true, (bool?)storageModel.Metadata["BoolDataProp"].Inner); - Assert.Equal(false, (bool?)storageModel.Metadata["NullableBoolDataProp"].Inner); - Assert.Equal(s_taglist, ((IEnumerable?)(storageModel.Metadata["TagListDataProp"].Inner!)) - .Select(x => x.Inner as string) + Assert.Equal(1, (double?)storageModel.Metadata["IntDataProp"]!.Value); + Assert.Equal(2, (double?)storageModel.Metadata["NullableIntDataProp"]!.Value); + Assert.Equal(3L, (double?)storageModel.Metadata["LongDataProp"]!.Value); + Assert.Equal(4L, (double?)storageModel.Metadata["NullableLongDataProp"]!.Value); + Assert.Equal(5.0f, (double?)storageModel.Metadata["FloatDataProp"]!.Value); + Assert.Equal(6.0f, (double?)storageModel.Metadata["NullableFloatDataProp"]!.Value); + Assert.Equal(7.0, (double?)storageModel.Metadata["DoubleDataProp"]!.Value); + Assert.Equal(8.0, (double?)storageModel.Metadata["NullableDoubleDataProp"]!.Value); + Assert.Equal(true, (bool?)storageModel.Metadata["BoolDataProp"]!.Value); + Assert.Equal(false, (bool?)storageModel.Metadata["NullableBoolDataProp"]!.Value); + Assert.Equal(s_taglist, ((IEnumerable?)(storageModel.Metadata["TagListDataProp"]!.Value!)) + .Select(x => x.Value as string) .ToArray()); Assert.Equal(s_vector, storageModel.Values); } @@ -136,9 +136,9 @@ public void MapFromDataToStorageModelMapsNullValues() // Assert Assert.Equal(TestKeyString, storageModel.Id); - Assert.True(storageModel.Metadata!["StringDataProp"].Inner == null); - Assert.True(storageModel.Metadata["NullableIntDataProp"].Inner == null); - Assert.True(storageModel.Metadata["NullableTagListDataProp"].Inner == null); + Assert.Null(storageModel.Metadata!["StringDataProp"]); + Assert.Null(storageModel.Metadata["NullableIntDataProp"]); + Assert.Null(storageModel.Metadata["NullableTagListDataProp"]); } [Fact] @@ -153,7 +153,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes() var storageModel = new Vector() { Id = TestKeyString, - Metadata = new MetadataMap() + Metadata = new Metadata() { ["StringDataProp"] = (MetadataValue)"string", ["IntDataProp"] = (MetadataValue)1, @@ -168,7 +168,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes() ["NullableBoolDataProp"] = (MetadataValue)false, ["TagListDataProp"] = (MetadataValue)new MetadataValue[] { "tag1", "tag2" } }, - Values = [1.0f, 2.0f, 3.0f] + Values = new float[] { 1.0f, 2.0f, 3.0f } }; // Act @@ -210,13 +210,13 @@ public void MapFromStorageToDataModelMapsNullValues() var storageModel = new Vector() { Id = TestKeyString, - Metadata = new MetadataMap() + Metadata = new Metadata() { - ["StringDataProp"] = new MetadataValue(), - ["NullableIntDataProp"] = new MetadataValue(), - ["NullableTagListDataProp"] = new MetadataValue(), + ["StringDataProp"] = null, + ["NullableIntDataProp"] = null, + ["NullableTagListDataProp"] = null, }, - Values = [1.0f, 2.0f, 3.0f] + Values = new float[] { 1.0f, 2.0f, 3.0f } }; var reader = new VectorStoreRecordPropertyReader( @@ -327,7 +327,7 @@ public void MapFromStorageToDataModelSkipsMissingProperties() var storageModel = new Vector() { Id = TestKeyString, - Values = [1.0f, 2.0f, 3.0f] + Values = new float[] { 1.0f, 2.0f, 3.0f } }; // Act diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs index 6777470e537d..21f7b6649da5 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeKernelBuilderExtensionsTests.cs @@ -26,7 +26,7 @@ public PineconeKernelBuilderExtensionsTests() public void AddVectorStoreRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._kernelBuilder.Services.AddSingleton(client); // Act. @@ -50,7 +50,7 @@ public void AddVectorStoreWithApiKeyRegistersClass() public void AddVectorStoreRecordCollectionRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._kernelBuilder.Services.AddSingleton(client); // Act. diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs index 084222d37bda..736cc3e3839d 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeServiceCollectionExtensionsTests.cs @@ -26,7 +26,7 @@ public PineconeServiceCollectionExtensionsTests() public void AddVectorStoreRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._serviceCollection.AddSingleton(client); // Act. @@ -49,7 +49,7 @@ public void AddVectorStoreWithApiKeyRegistersClass() public void AddVectorStoreRecordCollectionRegistersClass() { // Arrange. - using var client = new Sdk.PineconeClient("fake api key"); + var client = new Sdk.PineconeClient("fake api key"); this._serviceCollection.AddSingleton(client); // Act. diff --git a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs index 85ed14f7a468..0dc2620140f3 100644 --- a/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs +++ b/dotnet/src/Connectors/Connectors.Pinecone.UnitTests/PineconeVectorStoreRecordCollectionTests.cs @@ -35,7 +35,7 @@ public void CanCreateCollectionWithMismatchedDefinitionAndType() new VectorStoreRecordVectorProperty("Embedding", typeof(ReadOnlyMemory)) { Dimensions = 4 }, } }; - using var pineconeClient = new Sdk.PineconeClient("fake api key"); + var pineconeClient = new Sdk.PineconeClient("fake api key"); // Act. var sut = new PineconeVectorStoreRecordCollection( From 6918cfa108af79bc1647da21cf3577f9150c28d7 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 10 Mar 2025 17:10:11 +0100 Subject: [PATCH 7/8] remove all usages of Pinecone.NET --- dotnet/Directory.Packages.props | 1 - .../Process.IntegrationTestRunner.Dapr.csproj | 1 - ...ineconeVectorStoreRecordCollectionTests.cs | 42 -- .../Memory/Pinecone/PineconeAllTypes.cs | 64 -- .../Memory/Pinecone/PineconeHotel.cs | 40 - .../Pinecone/PineconeUserSecretsExtensions.cs | 37 - .../Pinecone/PineconeVectorStoreFixture.cs | 350 --------- ...ineconeVectorStoreRecordCollectionTests.cs | 684 ------------------ .../Pinecone/PineconeVectorStoreTests.cs | 54 -- .../PineconeApiKeySetConditionAttribute.cs | 21 - .../IntegrationTests/IntegrationTests.csproj | 2 - 11 files changed, 1296 deletions(-) delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs delete mode 100644 dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs diff --git a/dotnet/Directory.Packages.props b/dotnet/Directory.Packages.props index 3d3d6f558a5e..62c5b7c38946 100644 --- a/dotnet/Directory.Packages.props +++ b/dotnet/Directory.Packages.props @@ -65,7 +65,6 @@ - diff --git a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj index ba07a9a9dcad..2d35183b3648 100644 --- a/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj +++ b/dotnet/src/Experimental/Process.IntegrationTestRunner.Dapr/Process.IntegrationTestRunner.Dapr.csproj @@ -32,7 +32,6 @@ - diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs deleted file mode 100644 index 21964a61e3d0..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/CommonPineconeVectorStoreRecordCollectionTests.cs +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; -using Xunit; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -/// -/// Inherits common integration tests that should pass for any . -/// -/// Pinecone setup and teardown. -[Collection("PineconeVectorStoreTests")] -[PineconeApiKeySetCondition] -public class CommonPineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : BaseVectorStoreRecordCollectionTests, IClassFixture -{ - protected override string Key1 => "1"; - protected override string Key2 => "2"; - protected override string Key3 => "3"; - protected override string Key4 => "4"; - - protected override int DelayAfterIndexCreateInMilliseconds => 2000; - - protected override int DelayAfterUploadInMilliseconds => 15000; - - [SuppressMessage("Globalization", "CA1308:Normalize strings to uppercase", Justification = "Pinecone collection names should be lower case.")] - protected override IVectorStoreRecordCollection GetTargetRecordCollection(string recordCollectionName, VectorStoreRecordDefinition? vectorStoreRecordDefinition) - { - return new PineconeVectorStoreRecordCollection(fixture.Client, recordCollectionName.ToLowerInvariant(), new() - { - VectorStoreRecordDefinition = vectorStoreRecordDefinition - }); - } - - protected override HashSet GetSupportedDistanceFunctions() - { - return [DistanceFunction.CosineSimilarity, DistanceFunction.DotProductSimilarity, DistanceFunction.EuclideanSquaredDistance]; - } -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs deleted file mode 100644 index 7067781987bc..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeAllTypes.cs +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using Microsoft.Extensions.VectorData; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. -public record PineconeAllTypes() -{ - [VectorStoreRecordKey] - public string Id { get; init; } - - [VectorStoreRecordData] - public bool BoolProperty { get; set; } - [VectorStoreRecordData] - public bool? NullableBoolProperty { get; set; } - [VectorStoreRecordData] - public string StringProperty { get; set; } - [VectorStoreRecordData] - public string? NullableStringProperty { get; set; } - [VectorStoreRecordData] - public int IntProperty { get; set; } - [VectorStoreRecordData] - public int? NullableIntProperty { get; set; } - [VectorStoreRecordData] - public long LongProperty { get; set; } - [VectorStoreRecordData] - public long? NullableLongProperty { get; set; } - [VectorStoreRecordData] - public float FloatProperty { get; set; } - [VectorStoreRecordData] - public float? NullableFloatProperty { get; set; } - [VectorStoreRecordData] - public double DoubleProperty { get; set; } - [VectorStoreRecordData] - public double? NullableDoubleProperty { get; set; } - [VectorStoreRecordData] - public decimal DecimalProperty { get; set; } - [VectorStoreRecordData] - public decimal? NullableDecimalProperty { get; set; } - -#pragma warning disable CA1819 // Properties should not return arrays - [VectorStoreRecordData] - public string[] StringArray { get; set; } - [VectorStoreRecordData] - public string[]? NullableStringArray { get; set; } -#pragma warning restore CA1819 // Properties should not return arrays - - [VectorStoreRecordData] - public List StringList { get; set; } - [VectorStoreRecordData] - public List? NullableStringList { get; set; } - - [VectorStoreRecordData] - public IReadOnlyCollection Collection { get; set; } - [VectorStoreRecordData] - public IEnumerable Enumerable { get; set; } - - [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] - public ReadOnlyMemory? Embedding { get; set; } -} -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs deleted file mode 100644 index 54185830d5c0..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeHotel.cs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Text.Json.Serialization; -using Microsoft.Extensions.VectorData; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -#pragma warning disable CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. -public record PineconeHotel() - -{ - [VectorStoreRecordKey] - public string HotelId { get; init; } - - [VectorStoreRecordData] - public string HotelName { get; set; } - - [JsonPropertyName("code_of_the_hotel")] - [VectorStoreRecordData] - public int HotelCode { get; set; } - - [VectorStoreRecordData] - public float HotelRating { get; set; } - - [JsonPropertyName("json_parking")] - [VectorStoreRecordData(StoragePropertyName = "parking_is_included")] - public bool ParkingIncluded { get; set; } - - [VectorStoreRecordData] - public List Tags { get; set; } = []; - - [VectorStoreRecordData] - public string Description { get; set; } - - [VectorStoreRecordVector(Dimensions: 8, DistanceFunction: DistanceFunction.DotProductSimilarity)] - public ReadOnlyMemory DescriptionEmbedding { get; set; } -} -#pragma warning restore CS8618 // Non-nullable field must contain a non-null value when exiting constructor. Consider adding the 'required' modifier or declaring as nullable. diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs deleted file mode 100644 index 1644b7427e99..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeUserSecretsExtensions.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.IO; -using System.Reflection; -using System.Text.Json; -using Microsoft.Extensions.Configuration.UserSecrets; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; -public static class PineconeUserSecretsExtensions -{ - public const string PineconeApiKeyUserSecretEntry = "PineconeApiKey"; - - public static string ReadPineconeApiKey() - => JsonSerializer.Deserialize>( - File.ReadAllText(PathHelper.GetSecretsPathFromSecretsId( - typeof(PineconeUserSecretsExtensions).Assembly.GetCustomAttribute()! - .UserSecretsId)))![PineconeApiKeyUserSecretEntry].Trim(); - - public static bool ContainsPineconeApiKey() - { - var userSecretsIdAttribute = typeof(PineconeUserSecretsExtensions).Assembly.GetCustomAttribute(); - if (userSecretsIdAttribute == null) - { - return false; - } - - var path = PathHelper.GetSecretsPathFromSecretsId(userSecretsIdAttribute.UserSecretsId); - if (!File.Exists(path)) - { - return false; - } - - return JsonSerializer.Deserialize>( - File.ReadAllText(path))!.ContainsKey(PineconeApiKeyUserSecretEntry); - } -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs deleted file mode 100644 index c0c002f22ba0..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreFixture.cs +++ /dev/null @@ -1,350 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Net.Http; -using System.Text.RegularExpressions; -using System.Threading.Tasks; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using Pinecone.Grpc; -using Xunit; -using Sdk = Pinecone; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -public class PineconeVectorStoreFixture : IAsyncLifetime -{ - private const int MaxAttemptCount = 100; - private const int DelayInterval = 300; - - public string IndexName { get; } = "sk-index" -#pragma warning disable CA1308 // Normalize strings to uppercase - + new Regex("[^a-zA-Z0-9]", RegexOptions.None, matchTimeout: new TimeSpan(0, 0, 10)).Replace(Environment.MachineName.ToLowerInvariant(), ""); -#pragma warning restore CA1308 // Normalize strings to uppercase - - public Sdk.PineconeClient Client { get; private set; } = null!; - public PineconeVectorStore VectorStore { get; private set; } = null!; - public PineconeVectorStoreRecordCollection HotelRecordCollection { get; set; } = null!; - public PineconeVectorStoreRecordCollection AllTypesRecordCollection { get; set; } = null!; - public PineconeVectorStoreRecordCollection HotelRecordCollectionWithCustomNamespace { get; set; } = null!; - public IVectorStoreRecordCollection HotelRecordCollectionFromVectorStore { get; set; } = null!; - public IVectorStoreRecordCollection> HotelRecordCollectionWithGenericDataModel { get; set; } = null!; - - public virtual Sdk.Index Index { get; set; } = null!; - - public virtual async Task InitializeAsync() - { - this.Client = new Sdk.PineconeClient(PineconeUserSecretsExtensions.ReadPineconeApiKey()); - this.VectorStore = new PineconeVectorStore(this.Client); - - var hotelRecordDefinition = new VectorStoreRecordDefinition - { - Properties = - [ - new VectorStoreRecordKeyProperty(nameof(PineconeHotel.HotelId), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelName), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelCode), typeof(int)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.ParkingIncluded), typeof(bool)) { StoragePropertyName = "parking_is_included" }, - new VectorStoreRecordDataProperty(nameof(PineconeHotel.HotelRating), typeof(float)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.Tags), typeof(List)), - new VectorStoreRecordDataProperty(nameof(PineconeHotel.Description), typeof(string)), - new VectorStoreRecordVectorProperty(nameof(PineconeHotel.DescriptionEmbedding), typeof(ReadOnlyMemory)) { Dimensions = 8, DistanceFunction = DistanceFunction.DotProductSimilarity } - ] - }; - - var allTypesRecordDefinition = new VectorStoreRecordDefinition - { - Properties = - [ - new VectorStoreRecordKeyProperty(nameof(PineconeAllTypes.Id), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.BoolProperty), typeof(bool)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableBoolProperty), typeof(bool?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringProperty), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringProperty), typeof(string)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.IntProperty), typeof(int)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableIntProperty), typeof(int?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.LongProperty), typeof(long)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableLongProperty), typeof(long?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.FloatProperty), typeof(float)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableFloatProperty), typeof(float?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DoubleProperty), typeof(double)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDoubleProperty), typeof(double?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.DecimalProperty), typeof(decimal)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableDecimalProperty), typeof(decimal?)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringArray), typeof(string[])), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringArray), typeof(string[])), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.StringList), typeof(List)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.NullableStringList), typeof(List)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.Collection), typeof(IReadOnlyCollection)), - new VectorStoreRecordDataProperty(nameof(PineconeAllTypes.Enumerable), typeof(IEnumerable)), - new VectorStoreRecordVectorProperty(nameof(PineconeAllTypes.Embedding), typeof(ReadOnlyMemory?)) { Dimensions = 8, DistanceFunction = DistanceFunction.DotProductSimilarity } - ] - }; - - this.HotelRecordCollection = new PineconeVectorStoreRecordCollection( - this.Client, - this.IndexName, - new PineconeVectorStoreRecordCollectionOptions - { - VectorStoreRecordDefinition = hotelRecordDefinition - }); - - this.AllTypesRecordCollection = new PineconeVectorStoreRecordCollection( - this.Client, - this.IndexName, - new PineconeVectorStoreRecordCollectionOptions - { - VectorStoreRecordDefinition = allTypesRecordDefinition - }); - - this.HotelRecordCollectionWithCustomNamespace = new PineconeVectorStoreRecordCollection( - this.Client, - this.IndexName, - new PineconeVectorStoreRecordCollectionOptions - { - VectorStoreRecordDefinition = hotelRecordDefinition, - IndexNamespace = "my-namespace" - }); - - this.HotelRecordCollectionFromVectorStore = this.VectorStore.GetCollection( - this.IndexName, - hotelRecordDefinition); - - this.HotelRecordCollectionWithGenericDataModel = this.VectorStore.GetCollection>( - this.IndexName, - hotelRecordDefinition); - - await this.ClearIndexesAsync(); - await this.CreateIndexAndWaitAsync(); - await this.AddSampleDataAsync(); - } - - private async Task CreateIndexAndWaitAsync() - { - var attemptCount = 0; - - await this.HotelRecordCollection.CreateCollectionAsync(); - - do - { - await Task.Delay(DelayInterval); - attemptCount++; - this.Index = await this.Client.GetIndex(this.IndexName); - } while (!this.Index.Status.IsReady && attemptCount <= MaxAttemptCount); - - if (!this.Index.Status.IsReady) - { - throw new InvalidOperationException("'Create index' operation didn't complete in time. Index name: " + this.IndexName); - } - } - - public async Task DisposeAsync() - { - if (this.Client is not null) - { - await this.ClearIndexesAsync(); - this.Client.Dispose(); - } - } - - private async Task AddSampleDataAsync() - { - var fiveSeasons = new PineconeHotel - { - HotelId = "five-seasons", - HotelName = "Five Seasons Hotel", - Description = "Great service any season.", - HotelCode = 7, - HotelRating = 4.5f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([7.5f, 71.0f, 71.5f, 72.0f, 72.5f, 73.0f, 73.5f, 74.0f]), - Tags = ["wi-fi", "sauna", "gym", "pool"] - }; - - var vacationInn = new PineconeHotel - { - HotelId = "vacation-inn", - HotelName = "Vacation Inn Hotel", - Description = "On vacation? Stay with us.", - HotelCode = 11, - HotelRating = 4.3f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]), - Tags = ["wi-fi", "breakfast", "gym"] - }; - - var bestEastern = new PineconeHotel - { - HotelId = "best-eastern", - HotelName = "Best Eastern Hotel", - Description = "Best hotel east of New York.", - HotelCode = 42, - HotelRating = 4.7f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([47.5f, 421.0f, 741.5f, 744.0f, 742.5f, 483.0f, 743.5f, 744.0f]), - Tags = ["wi-fi", "breakfast", "gym"] - }; - - var stats = await this.Index.DescribeStats(); - var vectorCountBefore = stats.TotalVectorCount; - - // use both Upsert and BatchUpsert methods and also use record collections created directly and using vector store - await this.HotelRecordCollection.UpsertAsync(fiveSeasons); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - - await this.HotelRecordCollectionFromVectorStore.UpsertBatchAsync([vacationInn, bestEastern]).ToListAsync(); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 2); - - var allTypes1 = new PineconeAllTypes - { - Id = "all-types-1", - BoolProperty = true, - NullableBoolProperty = false, - StringProperty = "string prop 1", - NullableStringProperty = "nullable prop 1", - IntProperty = 1, - NullableIntProperty = 10, - LongProperty = 100L, - NullableLongProperty = 1000L, - FloatProperty = 10.5f, - NullableFloatProperty = 100.5f, - DoubleProperty = 23.75d, - NullableDoubleProperty = 233.75d, - DecimalProperty = 50.75m, - NullableDecimalProperty = 500.75m, - StringArray = ["one", "two"], - NullableStringArray = ["five", "six"], - StringList = ["eleven", "twelve"], - NullableStringList = ["fifteen", "sixteen"], - Collection = ["Foo", "Bar"], - Enumerable = ["another", "and another"], - Embedding = new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]) - }; - - var allTypes2 = new PineconeAllTypes - { - Id = "all-types-2", - BoolProperty = false, - NullableBoolProperty = null, - StringProperty = "string prop 2", - NullableStringProperty = null, - IntProperty = 2, - NullableIntProperty = null, - LongProperty = 200L, - NullableLongProperty = null, - FloatProperty = 20.5f, - NullableFloatProperty = null, - DoubleProperty = 43.75, - NullableDoubleProperty = null, - DecimalProperty = 250.75M, - NullableDecimalProperty = null, - StringArray = [], - NullableStringArray = null, - StringList = [], - NullableStringList = null, - Collection = [], - Enumerable = [], - Embedding = new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]) - }; - - await this.AllTypesRecordCollection.UpsertBatchAsync([allTypes1, allTypes2]).ToListAsync(); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 2); - - var custom = new PineconeHotel - { - HotelId = "custom-hotel", - HotelName = "Custom Hotel", - Description = "Everything customizable!", - HotelCode = 17, - HotelRating = 4.25f, - ParkingIncluded = true, - DescriptionEmbedding = new ReadOnlyMemory([147.5f, 1421.0f, 1741.5f, 1744.0f, 1742.5f, 1483.0f, 1743.5f, 1744.0f]), - }; - - await this.HotelRecordCollectionWithCustomNamespace.UpsertAsync(custom); - vectorCountBefore = await this.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - } - - public async Task VerifyVectorCountModifiedAsync(uint vectorCountBefore, int delta) - { - var attemptCount = 0; - Sdk.IndexStats stats; - - do - { - await Task.Delay(DelayInterval); - attemptCount++; - stats = await this.Index.DescribeStats(); - } while (stats.TotalVectorCount != vectorCountBefore + delta && attemptCount <= MaxAttemptCount); - - if (stats.TotalVectorCount != vectorCountBefore + delta) - { - throw new InvalidOperationException("'Upsert'/'Delete' operation didn't complete in time."); - } - - return stats.TotalVectorCount; - } - - public async Task DeleteAndWaitAsync(IEnumerable ids, string? indexNamespace = null) - { - var stats = await this.Index.DescribeStats(); - var vectorCountBefore = stats.Namespaces.Single(x => x.Name == (indexNamespace ?? "")).VectorCount; - var idCount = ids.Count(); - - var attemptCount = 0; - await this.Index.Delete(ids, indexNamespace); - long vectorCount; - do - { - await Task.Delay(DelayInterval); - attemptCount++; - stats = await this.Index.DescribeStats(); - vectorCount = stats.Namespaces.Single(x => x.Name == (indexNamespace ?? "")).VectorCount; - } while (vectorCount > vectorCountBefore - idCount && attemptCount <= MaxAttemptCount); - - if (vectorCount > vectorCountBefore - idCount) - { - throw new InvalidOperationException("'Delete' operation didn't complete in time."); - } - } - - private async Task ClearIndexesAsync() - { - var indexes = await this.Client.ListIndexes(); - var deletions = indexes.Select(x => this.DeleteExistingIndexAndWaitAsync(x.Name)); - - await Task.WhenAll(deletions); - } - - private async Task DeleteExistingIndexAndWaitAsync(string indexName) - { - var exists = true; - try - { - var attemptCount = 0; - await this.Client.DeleteIndex(indexName); - - do - { - await Task.Delay(DelayInterval); - var indexes = (await this.Client.ListIndexes()).Select(x => x.Name).ToArray(); - if (indexes.Length == 0 || !indexes.Contains(indexName)) - { - exists = false; - } - } while (exists && attemptCount <= MaxAttemptCount); - } - catch (HttpRequestException ex) when (ex.Message.Contains("NOT_FOUND")) - { - // index was already deleted - exists = false; - } - - if (exists) - { - throw new InvalidOperationException("'Delete index' operation didn't complete in time. Index name: " + indexName); - } - } -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs deleted file mode 100644 index 9b68eaf8d863..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreRecordCollectionTests.cs +++ /dev/null @@ -1,684 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Linq; -using System.Net; -using System.Net.Http; -using System.Threading.Tasks; -using Grpc.Core; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using Pinecone; -using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; -using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit; -using Xunit; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -#pragma warning disable CS0618 // VectorSearchFilter is obsolete - -[Collection("PineconeVectorStoreTests")] -[PineconeApiKeySetCondition] -public class PineconeVectorStoreRecordCollectionTests(PineconeVectorStoreFixture fixture) : IClassFixture -{ - private PineconeVectorStoreFixture Fixture { get; } = fixture; - - [VectorStoreFact] - public async Task TryCreateExistingIndexIsNoopAsync() - { - await this.Fixture.HotelRecordCollection.CreateCollectionIfNotExistsAsync(); - } - - [VectorStoreFact] - public async Task CollectionExistsReturnsTrueForExistingCollectionAsync() - { - var result = await this.Fixture.HotelRecordCollection.CollectionExistsAsync(); - - Assert.True(result); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task BasicGetAsync(bool includeVectors) - { - var fiveSeasons = await this.Fixture.HotelRecordCollection.GetAsync("five-seasons", new GetRecordOptions { IncludeVectors = includeVectors }); - - Assert.NotNull(fiveSeasons); - Assert.Equal("five-seasons", fiveSeasons.HotelId); - Assert.Equal("Five Seasons Hotel", fiveSeasons.HotelName); - Assert.Equal("Great service any season.", fiveSeasons.Description); - Assert.Equal(7, fiveSeasons.HotelCode); - Assert.Equal(4.5f, fiveSeasons.HotelRating); - Assert.True(fiveSeasons.ParkingIncluded); - Assert.Contains("wi-fi", fiveSeasons.Tags); - Assert.Contains("sauna", fiveSeasons.Tags); - Assert.Contains("gym", fiveSeasons.Tags); - Assert.Contains("pool", fiveSeasons.Tags); - - if (includeVectors) - { - Assert.Equal(new ReadOnlyMemory([7.5f, 71.0f, 71.5f, 72.0f, 72.5f, 73.0f, 73.5f, 74.0f]), fiveSeasons.DescriptionEmbedding); - } - else - { - Assert.Equal(new ReadOnlyMemory([]), fiveSeasons.DescriptionEmbedding); - } - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task BatchGetAsync(bool collectionFromVectorStore) - { - var hotelsCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollection - : this.Fixture.HotelRecordCollectionFromVectorStore; - - var hotels = await hotelsCollection.GetBatchAsync(["five-seasons", "vacation-inn", "best-eastern"]).ToListAsync(); - - var fiveSeasons = hotels.Single(x => x.HotelId == "five-seasons"); - var vacationInn = hotels.Single(x => x.HotelId == "vacation-inn"); - var bestEastern = hotels.Single(x => x.HotelId == "best-eastern"); - - Assert.Equal("Five Seasons Hotel", fiveSeasons.HotelName); - Assert.Equal("Great service any season.", fiveSeasons.Description); - Assert.Equal(7, fiveSeasons.HotelCode); - Assert.Equal(4.5f, fiveSeasons.HotelRating); - Assert.True(fiveSeasons.ParkingIncluded); - Assert.Contains("wi-fi", fiveSeasons.Tags); - Assert.Contains("sauna", fiveSeasons.Tags); - Assert.Contains("gym", fiveSeasons.Tags); - Assert.Contains("pool", fiveSeasons.Tags); - - Assert.Equal("Vacation Inn Hotel", vacationInn.HotelName); - Assert.Equal("On vacation? Stay with us.", vacationInn.Description); - Assert.Equal(11, vacationInn.HotelCode); - Assert.Equal(4.3f, vacationInn.HotelRating); - Assert.True(vacationInn.ParkingIncluded); - Assert.Contains("wi-fi", vacationInn.Tags); - Assert.Contains("breakfast", vacationInn.Tags); - Assert.Contains("gym", vacationInn.Tags); - - Assert.Equal("Best Eastern Hotel", bestEastern.HotelName); - Assert.Equal("Best hotel east of New York.", bestEastern.Description); - Assert.Equal(42, bestEastern.HotelCode); - Assert.Equal(4.7f, bestEastern.HotelRating); - Assert.True(bestEastern.ParkingIncluded); - Assert.Contains("wi-fi", bestEastern.Tags); - Assert.Contains("breakfast", bestEastern.Tags); - Assert.Contains("gym", bestEastern.Tags); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task AllTypesBatchGetAsync(bool includeVectors) - { - var allTypes = await this.Fixture.AllTypesRecordCollection.GetBatchAsync(["all-types-1", "all-types-2"], new GetRecordOptions { IncludeVectors = includeVectors }).ToListAsync(); - - var allTypes1 = allTypes.Single(x => x.Id == "all-types-1"); - var allTypes2 = allTypes.Single(x => x.Id == "all-types-2"); - - Assert.True(allTypes1.BoolProperty); - Assert.Equal("string prop 1", allTypes1.StringProperty); - Assert.Equal(1, allTypes1.IntProperty); - Assert.Equal(100L, allTypes1.LongProperty); - Assert.Equal(10.5f, allTypes1.FloatProperty); - Assert.Equal(23.75d, allTypes1.DoubleProperty); - Assert.Equal(50.75m, allTypes1.DecimalProperty); - Assert.Contains("one", allTypes1.StringArray); - Assert.Contains("two", allTypes1.StringArray); - Assert.Contains("eleven", allTypes1.StringList); - Assert.Contains("twelve", allTypes1.StringList); - Assert.Contains("Foo", allTypes1.Collection); - Assert.Contains("Bar", allTypes1.Collection); - Assert.Contains("another", allTypes1.Enumerable); - Assert.Contains("and another", allTypes1.Enumerable); - - Assert.False(allTypes2.BoolProperty); - Assert.Equal("string prop 2", allTypes2.StringProperty); - Assert.Equal(2, allTypes2.IntProperty); - Assert.Equal(200L, allTypes2.LongProperty); - Assert.Equal(20.5f, allTypes2.FloatProperty); - Assert.Equal(43.75d, allTypes2.DoubleProperty); - Assert.Equal(250.75m, allTypes2.DecimalProperty); - Assert.Empty(allTypes2.StringArray); - Assert.Empty(allTypes2.StringList); - Assert.Empty(allTypes2.Collection); - Assert.Empty(allTypes2.Enumerable); - - if (includeVectors) - { - Assert.True(allTypes1.Embedding.HasValue); - Assert.Equal(new ReadOnlyMemory([1.5f, 2.5f, 3.5f, 4.5f, 5.5f, 6.5f, 7.5f, 8.5f]), allTypes1.Embedding.Value); - - Assert.True(allTypes2.Embedding.HasValue); - Assert.Equal(new ReadOnlyMemory([10.5f, 20.5f, 30.5f, 40.5f, 50.5f, 60.5f, 70.5f, 80.5f]), allTypes2.Embedding.Value); - } - else - { - Assert.Null(allTypes1.Embedding); - Assert.Null(allTypes2.Embedding); - } - } - - [VectorStoreFact] - public async Task BatchGetIncludingNonExistingRecordAsync() - { - var hotels = await this.Fixture.HotelRecordCollection.GetBatchAsync(["vacation-inn", "non-existing"]).ToListAsync(); - - Assert.Single(hotels); - var vacationInn = hotels.Single(x => x.HotelId == "vacation-inn"); - - Assert.Equal("Vacation Inn Hotel", vacationInn.HotelName); - Assert.Equal("On vacation? Stay with us.", vacationInn.Description); - Assert.Equal(11, vacationInn.HotelCode); - Assert.Equal(4.3f, vacationInn.HotelRating); - Assert.True(vacationInn.ParkingIncluded); - Assert.Contains("wi-fi", vacationInn.Tags); - Assert.Contains("breakfast", vacationInn.Tags); - Assert.Contains("gym", vacationInn.Tags); - } - - [VectorStoreFact] - public async Task GetNonExistingRecordAsync() - { - var result = await this.Fixture.HotelRecordCollection.GetAsync("non-existing"); - Assert.Null(result); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task GetFromCustomNamespaceAsync(bool includeVectors) - { - var custom = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("custom-hotel", new GetRecordOptions { IncludeVectors = includeVectors }); - - Assert.NotNull(custom); - Assert.Equal("custom-hotel", custom.HotelId); - Assert.Equal("Custom Hotel", custom.HotelName); - if (includeVectors) - { - Assert.Equal(new ReadOnlyMemory([147.5f, 1421.0f, 1741.5f, 1744.0f, 1742.5f, 1483.0f, 1743.5f, 1744.0f]), custom.DescriptionEmbedding); - } - else - { - Assert.Equal(new ReadOnlyMemory([]), custom.DescriptionEmbedding); - } - } - - [VectorStoreFact] - public async Task TryGetVectorLocatedInDefaultNamespaceButLookInCustomNamespaceAsync() - { - var badFiveSeasons = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("five-seasons"); - - Assert.Null(badFiveSeasons); - } - - [VectorStoreFact] - public async Task TryGetVectorLocatedInCustomNamespaceButLookInDefaultNamespaceAsync() - { - var badCustomHotel = await this.Fixture.HotelRecordCollection.GetAsync("custom-hotel"); - - Assert.Null(badCustomHotel); - } - - [VectorStoreFact] - public async Task DeleteNonExistingRecordAsync() - { - await this.Fixture.HotelRecordCollection.DeleteAsync("non-existing"); - } - - [VectorStoreFact] - public async Task TryDeleteExistingVectorLocatedInDefaultNamespaceButUseCustomNamespaceDoesNotDoAnythingAsync() - { - await this.Fixture.HotelRecordCollectionWithCustomNamespace.DeleteAsync("five-seasons"); - - var stillThere = await this.Fixture.HotelRecordCollection.GetAsync("five-seasons"); - Assert.NotNull(stillThere); - Assert.Equal("five-seasons", stillThere.HotelId); - } - - [VectorStoreFact] - public async Task TryDeleteExistingVectorLocatedInCustomNamespaceButUseDefaultNamespaceDoesNotDoAnythingAsync() - { - await this.Fixture.HotelRecordCollection.DeleteAsync("custom-hotel"); - - var stillThere = await this.Fixture.HotelRecordCollectionWithCustomNamespace.GetAsync("custom-hotel"); - Assert.NotNull(stillThere); - Assert.Equal("custom-hotel", stillThere.HotelId); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task InsertGetModifyDeleteVectorAsync(bool collectionFromVectorStore) - { - var langriSha = new PineconeHotel - { - HotelId = "langri-sha", - HotelName = "Langri-Sha Hotel", - Description = "Lorem ipsum", - HotelCode = 100, - HotelRating = 4.2f, - ParkingIncluded = false, - DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f]) - }; - - var stats = await this.Fixture.Index.DescribeStats(); - var vectorCountBefore = stats.TotalVectorCount; - - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - - // insert - await hotelRecordCollection.UpsertAsync(langriSha); - - vectorCountBefore = await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - - var inserted = await hotelRecordCollection.GetAsync("langri-sha", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(inserted); - Assert.Equal(langriSha.HotelName, inserted.HotelName); - Assert.Equal(langriSha.Description, inserted.Description); - Assert.Equal(langriSha.HotelCode, inserted.HotelCode); - Assert.Equal(langriSha.HotelRating, inserted.HotelRating); - Assert.Equal(langriSha.ParkingIncluded, inserted.ParkingIncluded); - Assert.Equal(langriSha.DescriptionEmbedding, inserted.DescriptionEmbedding); - - langriSha.Description += " dolor sit amet"; - langriSha.ParkingIncluded = true; - langriSha.DescriptionEmbedding = new ReadOnlyMemory([11f, 12f, 13f, 14f, 15f, 16f, 17f, 18f]); - - // update - await hotelRecordCollection.UpsertAsync(langriSha); - - // this is not great but no vectors are added so we can't query status for number of vectors like we do for insert/delete - await Task.Delay(2000); - - var updated = await hotelRecordCollection.GetAsync("langri-sha", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(updated); - Assert.Equal(langriSha.HotelName, updated.HotelName); - Assert.Equal(langriSha.Description, updated.Description); - Assert.Equal(langriSha.HotelCode, updated.HotelCode); - Assert.Equal(langriSha.HotelRating, updated.HotelRating); - Assert.Equal(langriSha.ParkingIncluded, updated.ParkingIncluded); - Assert.Equal(langriSha.DescriptionEmbedding, updated.DescriptionEmbedding); - - // delete - await hotelRecordCollection.DeleteAsync("langri-sha"); - - await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: -1); - } - - [VectorStoreTheory] - [InlineData(true, true)] - [InlineData(true, false)] - [InlineData(false, true)] - [InlineData(false, false)] - public async Task VectorizedSearchAsync(bool collectionFromVectorStore, bool includeVectors) - { - // Arrange. - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]); - - // Act. - var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { IncludeVectors = includeVectors }); - var searchResults = await actual.Results.ToListAsync(); - var searchResultRecord = searchResults.First().Record; - - Assert.Equal("Vacation Inn Hotel", searchResultRecord.HotelName); - Assert.Equal("On vacation? Stay with us.", searchResultRecord.Description); - Assert.Equal(11, searchResultRecord.HotelCode); - Assert.Equal(4.3f, searchResultRecord.HotelRating); - Assert.True(searchResultRecord.ParkingIncluded); - Assert.Contains("wi-fi", searchResultRecord.Tags); - Assert.Contains("breakfast", searchResultRecord.Tags); - Assert.Contains("gym", searchResultRecord.Tags); - Assert.Equal(includeVectors, searchResultRecord.DescriptionEmbedding.Length > 0); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task VectorizedSearchWithTopSkipAsync(bool collectionFromVectorStore) - { - // Arrange. - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]); - - // Act. - var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { Skip = 1, Top = 1 }); - var searchResults = await actual.Results.ToListAsync(); - Assert.Single(searchResults); - var searchResultRecord = searchResults.First().Record; - Assert.Equal("Best Eastern Hotel", searchResultRecord.HotelName); - } - - [VectorStoreTheory] - [InlineData(true)] - [InlineData(false)] - public async Task VectorizedSearchWithFilterAsync(bool collectionFromVectorStore) - { - // Arrange. - var hotelRecordCollection = collectionFromVectorStore - ? this.Fixture.HotelRecordCollectionFromVectorStore - : this.Fixture.HotelRecordCollection; - var searchVector = new ReadOnlyMemory([17.5f, 721.0f, 731.5f, 742.0f, 762.5f, 783.0f, 793.5f, 704.0f]); - - // Act. - var filter = new VectorSearchFilter().EqualTo(nameof(PineconeHotel.HotelCode), 42); - var actual = await hotelRecordCollection.VectorizedSearchAsync(searchVector, new() { Top = 1, OldFilter = filter }); - var searchResults = await actual.Results.ToListAsync(); - Assert.Single(searchResults); - var searchResultRecord = searchResults.First().Record; - Assert.Equal("Best Eastern Hotel", searchResultRecord.HotelName); - } - - [VectorStoreFact] - public async Task ItCanUpsertAndRetrieveUsingTheGenericMapperAsync() - { - var merryYacht = new VectorStoreGenericDataModel("merry-yacht") - { - Data = - { - ["HotelName"] = "Merry Yacht Hotel", - ["Description"] = "Stay afloat at the Merry Yacht Hotel", - ["HotelCode"] = 101, - ["HotelRating"] = 4.2f, - ["ParkingIncluded"] = true, - ["Tags"] = new[] { "wi-fi", "breakfast", "gym" } - }, - Vectors = - { - ["DescriptionEmbedding"] = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f]) - } - }; - - var stats = await this.Fixture.Index.DescribeStats(); - var vectorCountBefore = stats.TotalVectorCount; - - var hotelRecordCollection = this.Fixture.HotelRecordCollectionWithGenericDataModel; - - // insert - await hotelRecordCollection.UpsertAsync(merryYacht); - - vectorCountBefore = await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: 1); - - var inserted = await hotelRecordCollection.GetAsync("merry-yacht", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(inserted); - Assert.Equal(merryYacht.Data["HotelName"], inserted.Data["HotelName"]); - Assert.Equal(merryYacht.Data["Description"], inserted.Data["Description"]); - Assert.Equal(merryYacht.Data["HotelCode"], inserted.Data["HotelCode"]); - Assert.Equal(merryYacht.Data["HotelRating"], inserted.Data["HotelRating"]); - Assert.Equal(merryYacht.Data["ParkingIncluded"], inserted.Data["ParkingIncluded"]); - Assert.Equal(merryYacht.Data["Tags"], inserted.Data["Tags"]); - Assert.Equal( - ((ReadOnlyMemory)merryYacht.Vectors["DescriptionEmbedding"]!).ToArray(), - ((ReadOnlyMemory)inserted.Vectors["DescriptionEmbedding"]!).ToArray()); - - // delete - await hotelRecordCollection.DeleteAsync("merry-yacht"); - - await this.Fixture.VerifyVectorCountModifiedAsync(vectorCountBefore, delta: -1); - } - - [VectorStoreFact] - public async Task UseCollectionExistsOnNonExistingStoreReturnsFalseAsync() - { - var incorrectRecordStore = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "incorrect"); - - var result = await incorrectRecordStore.CollectionExistsAsync(); - - Assert.False(result); - } - - [VectorStoreFact] - public async Task UseNonExistingIndexThrowsAsync() - { - var incorrectRecordStore = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "incorrect"); - - var statusCode = (await Assert.ThrowsAsync( - () => incorrectRecordStore.GetAsync("best-eastern"))).StatusCode; - - Assert.Equal(HttpStatusCode.NotFound, statusCode); - } - - [VectorStoreFact] - public async Task UseRecordStoreWithCustomMapperAsync() - { - var recordStore = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - this.Fixture.IndexName, - new PineconeVectorStoreRecordCollectionOptions { VectorCustomMapper = new CustomHotelRecordMapper() }); - - var vacationInn = await recordStore.GetAsync("vacation-inn", new GetRecordOptions { IncludeVectors = true }); - - Assert.NotNull(vacationInn); - Assert.Equal("Custom Vacation Inn Hotel", vacationInn.HotelName); - Assert.Equal("On vacation? Stay with us.", vacationInn.Description); - Assert.Equal(11, vacationInn.HotelCode); - Assert.Equal(4.3f, vacationInn.HotelRating); - Assert.True(vacationInn.ParkingIncluded); - Assert.Contains("wi-fi", vacationInn.Tags); - Assert.Contains("breakfast", vacationInn.Tags); - Assert.Contains("gym", vacationInn.Tags); - } - - private sealed class CustomHotelRecordMapper : IVectorStoreRecordMapper - { - public Vector MapFromDataToStorageModel(PineconeHotel dataModel) - { - var metadata = new MetadataMap - { - [nameof(PineconeHotel.HotelName)] = dataModel.HotelName, - [nameof(PineconeHotel.Description)] = dataModel.Description, - [nameof(PineconeHotel.HotelCode)] = dataModel.HotelCode, - [nameof(PineconeHotel.HotelRating)] = dataModel.HotelRating, - ["parking_is_included"] = dataModel.ParkingIncluded, - [nameof(PineconeHotel.Tags)] = dataModel.Tags.ToArray(), - }; - - return new Vector - { - Id = dataModel.HotelId, - Values = dataModel.DescriptionEmbedding.ToArray(), - Metadata = metadata, - }; - } - - public PineconeHotel MapFromStorageToDataModel(Vector storageModel, StorageToDataModelMapperOptions options) - { - if (storageModel.Metadata == null) - { - throw new InvalidOperationException("Missing metadata."); - } - - return new PineconeHotel - { - HotelId = storageModel.Id, - HotelName = "Custom " + (string)storageModel.Metadata[nameof(PineconeHotel.HotelName)].Inner!, - Description = (string)storageModel.Metadata[nameof(PineconeHotel.Description)].Inner!, - HotelCode = (int)(double)storageModel.Metadata[nameof(PineconeHotel.HotelCode)].Inner!, - HotelRating = (float)(double)storageModel.Metadata[nameof(PineconeHotel.HotelRating)].Inner!, - ParkingIncluded = (bool)storageModel.Metadata["parking_is_included"].Inner!, - Tags = ((MetadataValue[])storageModel.Metadata[nameof(PineconeHotel.Tags)].Inner!)!.Select(x => (string)x.Inner!).ToList(), - }; - } - } - - #region Negative - - [VectorStoreFact] - public void UseRecordWithNoEmbeddingThrows() - { - var exception = Assert.Throws( - () => new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "Whatever")); - - Assert.Equal( - $"No vector property found on type {nameof(PineconeRecordNoEmbedding)} or the provided VectorStoreRecordDefinition while at least one is required.", - exception.Message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordNoEmbedding - { - [VectorStoreRecordKey] - public int Id { get; set; } - - [VectorStoreRecordData] - public string? Name { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public void UseRecordWithMultipleEmbeddingsThrows() - { - var exception = Assert.Throws( - () => new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "Whatever")); - - Assert.Equal( - $"Multiple vector properties found on type {nameof(PineconeRecordMultipleEmbeddings)} or the provided VectorStoreRecordDefinition while only one is supported.", - exception.Message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordMultipleEmbeddings - { - [VectorStoreRecordKey] - public string Id { get; set; } = null!; - - [VectorStoreRecordVector] - public ReadOnlyMemory Embedding1 { get; set; } - - [VectorStoreRecordVector] - public ReadOnlyMemory Embedding2 { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public void UseRecordWithUnsupportedKeyTypeThrows() - { - var message = Assert.Throws( - () => new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "Whatever")).Message; - - Assert.Equal( - $"Key properties must be one of the supported types: {typeof(string).FullName}. Type of the property '{nameof(PineconeRecordUnsupportedKeyType.Id)}' is {typeof(int).FullName}.", - message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordUnsupportedKeyType - { - [VectorStoreRecordKey] - public int Id { get; set; } - - [VectorStoreRecordData] - public string? Name { get; set; } - - [VectorStoreRecordVector] - public ReadOnlyMemory Embedding { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public async Task TryAddingVectorWithUnsupportedValuesAsync() - { - var badAllTypes = new PineconeAllTypes - { - Id = "bad", - BoolProperty = true, - DecimalProperty = 1m, - DoubleProperty = 1.5d, - FloatProperty = 2.5f, - IntProperty = 1, - LongProperty = 11L, - NullableStringArray = ["foo", null!, "bar",], - Embedding = new ReadOnlyMemory([1f, 2f, 3f, 4f, 5f, 6f, 7f, 8f]) - }; - - var exception = await Assert.ThrowsAsync( - () => this.Fixture.AllTypesRecordCollection.UpsertAsync(badAllTypes)); - - Assert.Equal("Microsoft.SemanticKernel.Connectors.Pinecone", exception.Source); - Assert.Equal("Pinecone", exception.VectorStoreType); - Assert.Equal("Upsert", exception.OperationName); - Assert.Equal(this.Fixture.IndexName, exception.CollectionName); - - var inner = exception.InnerException as RpcException; - Assert.NotNull(inner); - Assert.Equal(StatusCode.InvalidArgument, inner.StatusCode); - } - - [VectorStoreFact] - public async Task TryCreateIndexWithIncorrectDimensionFailsAsync() - { - var recordCollection = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "negative-dimension"); - - var message = (await Assert.ThrowsAsync(() => recordCollection.CreateCollectionAsync())).Message; - - Assert.Equal("Property Dimensions on VectorStoreRecordVectorProperty 'Embedding' must be set to a positive integer to create a collection.", message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordWithIncorrectDimension - { - [VectorStoreRecordKey] - public string Id { get; set; } = null!; - - [VectorStoreRecordData] - public string? Name { get; set; } - - [VectorStoreRecordVector(Dimensions: -7)] - public ReadOnlyMemory Embedding { get; set; } - } -#pragma warning restore CA1812 - - [VectorStoreFact] - public async Task TryCreateIndexWithUnsSupportedMetricFailsAsync() - { - var recordCollection = new PineconeVectorStoreRecordCollection( - this.Fixture.Client, - "bad-metric"); - - var message = (await Assert.ThrowsAsync(() => recordCollection.CreateCollectionAsync())).Message; - - Assert.Equal("Distance function 'just eyeball it' for VectorStoreRecordVectorProperty 'Embedding' is not supported by the Pinecone VectorStore.", message); - } - -#pragma warning disable CA1812 - private sealed record PineconeRecordWithUnsupportedMetric - { - [VectorStoreRecordKey] - public string Id { get; set; } = null!; - - [VectorStoreRecordData] - public string? Name { get; set; } - - [VectorStoreRecordVector(Dimensions: 5, DistanceFunction: "just eyeball it")] - public ReadOnlyMemory Embedding { get; set; } - } -#pragma warning restore CA1812 - - #endregion -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs deleted file mode 100644 index 4cd63ec6b8a9..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/PineconeVectorStoreTests.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using Microsoft.Extensions.VectorData; -using Microsoft.SemanticKernel.Connectors.Pinecone; -using SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; -using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit; -using Xunit; -using Sdk = Pinecone; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone; - -[Collection("PineconeVectorStoreTests")] -[PineconeApiKeySetCondition] -public class PineconeVectorStoreTests(PineconeVectorStoreFixture fixture) - : BaseVectorStoreTests(new PineconeVectorStore(fixture.Client)), IClassFixture -{ - private PineconeVectorStoreFixture Fixture { get; } = fixture; - -#pragma warning disable CS0618 // IPineconeVectorStoreRecordCollectionFactory is obsolete - [VectorStoreFact] - public void CreateCollectionUsingFactory() - { - var vectorStore = new PineconeVectorStore( - this.Fixture.Client, - new PineconeVectorStoreOptions - { - VectorStoreCollectionFactory = new MyVectorStoreRecordCollectionFactory() - }); - - var factoryCollection = vectorStore.GetCollection(this.Fixture.IndexName); - - Assert.NotNull(factoryCollection); - Assert.Equal("factory" + this.Fixture.IndexName, factoryCollection.CollectionName); - } - - private sealed class MyVectorStoreRecordCollectionFactory : IPineconeVectorStoreRecordCollectionFactory - { - public IVectorStoreRecordCollection CreateVectorStoreRecordCollection( - Sdk.PineconeClient pineconeClient, - string name, - VectorStoreRecordDefinition? vectorStoreRecordDefinition) - where TKey : notnull - { - if (typeof(TKey) != typeof(string)) - { - throw new InvalidOperationException("Only string keys are supported."); - } - - return (new PineconeVectorStoreRecordCollection(pineconeClient, "factory" + name) as IVectorStoreRecordCollection)!; - } - } -#pragma warning restore CS0618 -} diff --git a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs b/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs deleted file mode 100644 index b677c47c378f..000000000000 --- a/dotnet/src/IntegrationTests/Connectors/Memory/Pinecone/Xunit/PineconeApiKeySetConditionAttribute.cs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Threading.Tasks; -using SemanticKernel.IntegrationTests.Connectors.Memory.Xunit; - -namespace SemanticKernel.IntegrationTests.Connectors.Memory.Pinecone.Xunit; - -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class)] -public sealed class PineconeApiKeySetConditionAttribute : Attribute, ITestCondition -{ - public ValueTask IsMetAsync() - { - var isMet = PineconeUserSecretsExtensions.ContainsPineconeApiKey(); - - return ValueTask.FromResult(isMet); - } - - public string SkipReason - => $"Pinecone API key was not specified in user secrets. Use the following command to set it: dotnet user-secrets set \"{PineconeUserSecretsExtensions.PineconeApiKeyUserSecretEntry}\" \"your_Pinecone_API_key\""; -} diff --git a/dotnet/src/IntegrationTests/IntegrationTests.csproj b/dotnet/src/IntegrationTests/IntegrationTests.csproj index 6a75257b503a..033c0f57bd78 100644 --- a/dotnet/src/IntegrationTests/IntegrationTests.csproj +++ b/dotnet/src/IntegrationTests/IntegrationTests.csproj @@ -52,7 +52,6 @@ - @@ -69,7 +68,6 @@ - From 9911ee546273b64eb722e86e65e29d6383eea2bd Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 10 Mar 2025 17:11:57 +0100 Subject: [PATCH 8/8] tests: - Collection conformance tests: CRD for the collection itself - Record conformance tests: CRUD for single record - implement and fix them for Redis, PostgreSQL and SQL Server as well - VectorizedSearchAsync conformance tests --- .../Connectors.Memory.Pinecone.csproj | 4 ++ .../PineconeVectorStoreRecordCollection.cs | 53 +++++++++++--- .../RedisJsonVectorStoreRecordCollection.cs | 11 ++- .../CRUD/PineconeBatchConformanceTests.cs | 12 ++++ .../PineconeCollectionConformanceTests.cs | 12 ++++ ...ineconeGenericDataModelConformanceTests.cs | 12 ++++ .../CRUD/PineconeRecordConformanceTests.cs | 12 ++++ .../PineconeIntegrationTests/SampleTests.cs | 35 --------- .../Support/PineconeTestStore.cs | 2 + ...orSearchDistanceFunctionComplianceTests.cs | 26 +++++++ .../PostgresCollectionConformanceTests.cs | 12 ++++ .../CRUD/PostgresRecordConformanceTests.cs | 12 ++++ .../CRUD/RedisCollectionConformanceTests.cs | 12 ++++ .../RedisGenericDataModelConformanceTests.cs | 12 ++++ .../CRUD/RedisRecordConformanceTests.cs | 12 ++++ .../Support/RedisFixture.cs | 10 +++ .../SqlServerCollectionConformanceTests.cs | 12 ++++ .../CRUD/SqlServerRecordConformanceTests.cs | 12 ++++ .../CRUD/BatchConformanceTests.cs | 61 ++++++++++++++++ .../CRUD/CollectionConformanceTests.cs | 56 +++++++++++++++ .../CRUD/ConformanceTestsBase.cs | 63 +++++++++++++++- .../CRUD/GenericDataModelConformanceTests.cs | 12 +--- .../CRUD/RecordConformanceTests.cs | 72 +++++++++++++++++++ .../Models/SimpleModel.cs | 4 +- 24 files changed, 479 insertions(+), 62 deletions(-) create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs delete mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj index 3b51b03623ae..6f73bf6e5b55 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/Connectors.Memory.Pinecone.csproj @@ -18,6 +18,10 @@ Pinecone connector for Semantic Kernel plugins and semantic memory + + $(DefineConstants);NON_PUBLISH + + diff --git a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs index 8275a733d595..8442482961a5 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Pinecone/PineconeVectorStoreRecordCollection.cs @@ -6,7 +6,6 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using Grpc.Core; using Microsoft.Extensions.VectorData; using Pinecone; using Sdk = Pinecone; @@ -120,9 +119,14 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can { if (!await this.CollectionExistsAsync(cancellationToken).ConfigureAwait(false)) { - await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); - // If the collection already exists, we should ignore the exception. - // TODO adsitnik: find out which exception is thrown when the collection already exists. + try + { + await this.CreateCollectionAsync(cancellationToken).ConfigureAwait(false); + } + catch (VectorStoreOperationException ex) when (ex.InnerException is PineconeApiException apiEx && apiEx.InnerException is ConflictError) + { + // If the collection got created in the meantime, we should ignore the exception. + } } } @@ -130,7 +134,17 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = default) => this.RunOperationAsync( "DeleteCollection", - () => this._pineconeClient.DeleteIndexAsync(this.CollectionName, cancellationToken: cancellationToken)); + async () => + { + try + { + await this._pineconeClient.DeleteIndexAsync(this.CollectionName, cancellationToken: cancellationToken).ConfigureAwait(false); + } + catch (NotFoundError) + { + // If the collection does not exist, we should ignore the exception. + } + }); /// public virtual async Task GetAsync(string key, GetRecordOptions? options = null, CancellationToken cancellationToken = default) @@ -329,15 +343,13 @@ public virtual async Task> VectorizedSearchAsync this.GetIndexClient().QueryAsync(request, cancellationToken: cancellationToken)).ConfigureAwait(false); - if (response.Results is null) + if (response.Matches is null) { return new VectorSearchResults(Array.Empty>().ToAsyncEnumerable()); } // Pinecone does not provide a way to skip results, so we need to do it manually. - var skippedResults = response.Results - .Where(result => result.Matches is not null) - .SelectMany(result => result.Matches!) + var skippedResults = response.Matches .Skip(options.Skip); StorageToDataModelMapperOptions mapperOptions = new() { IncludeVectors = options.IncludeVectors is true }; @@ -363,7 +375,7 @@ private async Task RunOperationAsync(string operationName, Func> o { return await operation.Invoke().ConfigureAwait(false); } - catch (RpcException ex) + catch (PineconeApiException ex) { throw new VectorStoreOperationException("Call to vector store failed.", ex) { @@ -392,7 +404,26 @@ private async Task RunOperationAsync(string operationName, Func operation) } private IndexClient GetIndexClient() - => this._indexClient ??= this._pineconeClient.Index(name: this.CollectionName); + { + if (this._indexClient is null) + { +#if NON_PUBLISH + // When "host" is not provided for PineconeClient.Index, + // it will try to get the host from the Pinecone service. + // In the cloud environment it's fine, but with the local emulator + // it reports the address of the REST endpoint rather than the gRPC one. + // To work around this, we set the environment variable to the gRPC endpoint. + // But only for the non-publish build, so this logic is not used in production. + string? hostName = Environment.GetEnvironmentVariable("PINECONE_GRPC_ENDPOINT"); +#else + string? hostName = null; +#endif + + this._indexClient = this._pineconeClient.Index(name: this.CollectionName, hostName); + } + + return this._indexClient; + } private static ServerlessSpecCloud MapCloud(string serverlessIndexCloud) => serverlessIndexCloud switch diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs index 14a8e56222d9..52f6091f129e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs @@ -177,9 +177,16 @@ public virtual async Task CreateCollectionIfNotExistsAsync(CancellationToken can } /// - public virtual Task DeleteCollectionAsync(CancellationToken cancellationToken = default) + public virtual async Task DeleteCollectionAsync(CancellationToken cancellationToken = default) { - return this.RunOperationAsync("FT.DROPINDEX", () => this._database.FT().DropIndexAsync(this._collectionName)); + try + { + await this.RunOperationAsync("FT.DROPINDEX", () => this._database.FT().DropIndexAsync(this._collectionName)).ConfigureAwait(false); + } + catch (VectorStoreOperationException ex) when (ex.InnerException is RedisServerException innerEx && innerEx.Message.Contains("Unknown Index name")) + { + // Collection does not exist. + } } /// diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs new file mode 100644 index 000000000000..682e43c6d45a --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeBatchConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeBatchConformanceTests(PineconeFixture fixture) + : BatchConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs new file mode 100644 index 000000000000..45fbbb18dc64 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeCollectionConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeCollectionConformanceTests(PineconeFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs new file mode 100644 index 000000000000..0f43f43bec5c --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeGenericDataModelConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeGenericDataModelConformanceTests(PineconeFixture fixture) + : GenericDataModelConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs new file mode 100644 index 000000000000..9e4acddea020 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/CRUD/PineconeRecordConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PineconeIntegrationTests.CRUD; + +public class PineconeRecordConformanceTests(PineconeFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs deleted file mode 100644 index 89138368cc57..000000000000 --- a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/SampleTests.cs +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using Pinecone; -using PineconeIntegrationTests.Support; -using VectorDataSpecificationTests.Xunit; -using Xunit; - -namespace PineconeIntegrationTests; - -public class SampleTests(PineconeFixture fixture) : IClassFixture -{ - [ConditionalFact] - public async Task CanRunSampleCode() - { - const string IndexName = "sample-index-name"; - - await fixture.Client.CreateIndexAsync(new CreateIndexRequest - { - Name = IndexName, - Dimension = 2, - Metric = CreateIndexRequestMetric.Cosine, - Spec = new ServerlessIndexSpec - { - Serverless = new ServerlessSpec - { - Cloud = ServerlessSpecCloud.Aws, - Region = "us-east-1", - } - }, - DeletionProtection = DeletionProtection.Disabled, - }); - - await fixture.Client.DeleteIndexAsync(IndexName); - } -} diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs index bfedfe549d17..509c81c087ed 100644 --- a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/Support/PineconeTestStore.cs @@ -60,6 +60,8 @@ protected override async Task StartAsync() apiKey: "ForPineconeLocalTheApiKeysAreIgnored", clientOptions: clientOptions); + Environment.SetEnvironmentVariable("PINECONE_GRPC_ENDPOINT", grpcUrl); + this._defaultVectorStore = new(this._client); } diff --git a/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs new file mode 100644 index 000000000000..f7dfc8e76834 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PineconeIntegrationTests/VectorSearch/PineconeVectorSearchDistanceFunctionComplianceTests.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PineconeIntegrationTests.Support; +using VectorDataSpecificationTests.VectorSearch; +using Xunit; + +namespace PineconeIntegrationTests.VectorSearch; + +public class PineconeVectorSearchDistanceFunctionComplianceTests(PineconeFixture fixture) + : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture +{ + public override Task CosineDistance() + => Assert.ThrowsAsync(base.CosineDistance); + + public override Task EuclideanDistance() + => Assert.ThrowsAsync(base.EuclideanDistance); + + public override Task Hamming() + => Assert.ThrowsAsync(base.Hamming); + + public override Task ManhattanDistance() + => Assert.ThrowsAsync(base.ManhattanDistance); + + public override Task NegativeDotProductSimilarity() + => Assert.ThrowsAsync(base.NegativeDotProductSimilarity); +} diff --git a/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs new file mode 100644 index 000000000000..14815cb55af0 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresCollectionConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PostgresIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PostgresIntegrationTests.CRUD; + +public class PostgresCollectionConformanceTests(PostgresFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs new file mode 100644 index 000000000000..6f290168ead4 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresRecordConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PostgresIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace PostgresIntegrationTests.CRUD; + +public class PostgresRecordConformanceTests(PostgresFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs new file mode 100644 index 000000000000..cc2ade48d481 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisCollectionConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using RedisIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace RedisIntegrationTests.CRUD; + +public class RedisCollectionConformanceTests(RedisFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs new file mode 100644 index 000000000000..b0c4f13084bd --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisGenericDataModelConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using RedisIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace RedisIntegrationTests.CRUD; + +public class RedisGenericDataModelConformanceTests(RedisFixture fixture) + : GenericDataModelConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs new file mode 100644 index 000000000000..87b2d5a89e65 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisRecordConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using RedisIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace RedisIntegrationTests.CRUD; + +public class RedisRecordConformanceTests(RedisFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs new file mode 100644 index 000000000000..a8c7004eaec2 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/Support/RedisFixture.cs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.Support; + +namespace RedisIntegrationTests.Support; + +public class RedisFixture : VectorStoreFixture +{ + public override TestStore TestStore => RedisTestStore.Instance; +} diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs new file mode 100644 index 000000000000..acbe977e8fa7 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerCollectionConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using SqlServerIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace SqlServerIntegrationTests.CRUD; + +public class SqlServerCollectionConformanceTests(SqlServerFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs new file mode 100644 index 000000000000..96939b19db23 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerRecordConformanceTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using SqlServerIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using Xunit; + +namespace SqlServerIntegrationTests.CRUD; + +public class SqlServerRecordConformanceTests(SqlServerFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs index ace837591a74..ab83e58c8082 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs @@ -66,4 +66,65 @@ await this.ExecuteAsync(async collection => Assert.Equal("keys", ex.ParamName); }); } + + [ConditionalFact] + public Task CanInsertUpdateAndDelete_WithVectors() + => this.CanInsertUpdateAndDelete(includeVectors: true); + + [ConditionalFact] + public Task CanInsertUpdateAndDelete_WithoutVectors() + => this.CanInsertUpdateAndDelete(includeVectors: false); + + private async Task CanInsertUpdateAndDelete(bool includeVectors) + { + await this.ExecuteAsync(async collection => + { + SimpleModel[] inserted = Enumerable.Range(0, 10).Select(i => new SimpleModel() + { + Id = this.Fixture.GenerateNextKey(), + Number = 100 + i, + Text = i.ToString(), + Floats = Enumerable.Range(0, 10).Select(j => (float)(i + j)).ToArray() + }).ToArray(); + + TKey[] keys = await collection.UpsertBatchAsync(inserted).ToArrayAsync(); + Assert.Equal( + inserted.Select(r => r.Id).OrderBy(id => id).ToArray(), + keys.OrderBy(id => id).ToArray()); + + SimpleModel[] received = await collection.GetBatchAsync(keys, new() { IncludeVectors = includeVectors }).ToArrayAsync(); + for (int i = 0; i < inserted.Length; i++) + { + this.AssertEqual(inserted[i], this.GetRecord(received, inserted[i].Id!), includeVectors); + } + + SimpleModel[] updated = inserted.Select(i => new SimpleModel() + { + Id = i.Id, + Text = i.Text + "updated", + Number = i.Number + 200, + Floats = i.Floats + }).ToArray(); + + keys = await collection.UpsertBatchAsync(updated).ToArrayAsync(); + Assert.Equal( + updated.Select(r => r.Id).OrderBy(id => id).ToArray(), + keys.OrderBy(id => id).ToArray()); + + received = await collection.GetBatchAsync(keys, new() { IncludeVectors = includeVectors }).ToArrayAsync(); + for (int i = 0; i < updated.Length; i++) + { + this.AssertEqual(updated[i], this.GetRecord(received, updated[i].Id!), includeVectors); + } + + await collection.DeleteBatchAsync(keys); + + Assert.False(await collection.GetBatchAsync(keys).AnyAsync()); + }); + } + + // The order of records in the received array is not guaranteed + // to match the order of keys in the requested keys array. + private SimpleModel GetRecord(SimpleModel[] received, TKey key) + => received.Single(r => r.Id!.Equals(key)); } diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs new file mode 100644 index 000000000000..4e23b0c8a249 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/CollectionConformanceTests.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.VectorData; +using VectorDataSpecificationTests.Models; +using VectorDataSpecificationTests.Support; +using VectorDataSpecificationTests.Xunit; +using Xunit; + +namespace VectorDataSpecificationTests.CRUD; + +public class CollectionConformanceTests(VectorStoreFixture fixture) + : ConformanceTestsBase>(fixture) where TKey : notnull +{ + [ConditionalFact] + public async Task DeletingNonExistingCollectionDoesNotThrow() + { + await this.ExecuteAsync(async collection => + { + Assert.False(await collection.CollectionExistsAsync()); + + await collection.DeleteCollectionAsync(); + }, createCollection: false); + } + + [ConditionalFact] + public async Task CreateCollectionIfNotExistsCalledMoreThanOnceDoesNotThrow() + { + await this.ExecuteAsync(async collection => + { + Assert.False(await collection.CollectionExistsAsync()); + + await collection.CreateCollectionIfNotExistsAsync(); + Assert.True(await collection.CollectionExistsAsync()); + Assert.True(await this.Fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + + await collection.CreateCollectionIfNotExistsAsync(); + }, createCollection: false); + } + + [ConditionalFact] + public async Task CreateCollectionAsyncCalledMoreThanOnceThrows() + { + await this.ExecuteAsync(async collection => + { + Assert.False(await collection.CollectionExistsAsync()); + + await collection.CreateCollectionAsync(); + Assert.True(await collection.CollectionExistsAsync()); + Assert.True(await this.Fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); + + await collection.CreateCollectionIfNotExistsAsync(); + + await Assert.ThrowsAsync(() => collection.CreateCollectionAsync()); + }, createCollection: false); + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs index 21a6c95f8986..7dbaf04fbcba 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/ConformanceTestsBase.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. using Microsoft.Extensions.VectorData; +using VectorDataSpecificationTests.Models; using VectorDataSpecificationTests.Support; +using Xunit; namespace VectorDataSpecificationTests.CRUD; @@ -14,13 +16,16 @@ public abstract class ConformanceTestsBase(VectorStoreFixture fix protected virtual VectorStoreRecordDefinition? GetRecordDefinition() => null; - protected async Task ExecuteAsync(Func, Task> test) + protected async Task ExecuteAsync(Func, Task> test, bool createCollection = true) { string collectionName = this.GetUniqueCollectionName(); var collection = this.Fixture.TestStore.DefaultVectorStore.GetCollection(collectionName, this.GetRecordDefinition()); - await collection.CreateCollectionAsync(); + if (createCollection) + { + await collection.CreateCollectionAsync(); + } try { @@ -31,4 +36,58 @@ protected async Task ExecuteAsync(Func)) + { + var expectedSimple = (SimpleModel)(object)expected!; + var actualSimple = (SimpleModel)(object)actual!; + + Assert.Equal(expectedSimple!.Id, actualSimple!.Id); + Assert.Equal(expectedSimple.Text, actualSimple.Text); + Assert.Equal(expectedSimple.Number, actualSimple.Number); + + if (includeVectors) + { + Assert.Equal(expectedSimple.Floats.ToArray(), actualSimple.Floats.ToArray()); + } + else + { + Assert.Equal(0, actualSimple.Floats.Length); + } + } + else if (typeof(TRecord) == typeof(VectorStoreGenericDataModel)) + { + var expectedGeneric = (VectorStoreGenericDataModel)(object)expected!; + var actualGeneric = (VectorStoreGenericDataModel)(object)actual!; + + Assert.Equal(expectedGeneric!.Key, actualGeneric!.Key); + + foreach (var pair in expectedGeneric.Data) + { + Assert.Equal(pair.Value, actualGeneric.Data[pair.Key]); + } + + foreach (var pair in expectedGeneric.Vectors) + { + if (includeVectors) + { + Assert.Equal( + ((ReadOnlyMemory)pair.Value!).ToArray(), + ((ReadOnlyMemory)actualGeneric.Vectors[pair.Key]!).ToArray()); + } + else + { + Assert.Equal(0, ((ReadOnlyMemory)actualGeneric.Vectors[pair.Key]!).Length); + } + } + } + else + { + throw new NotSupportedException($"Type {typeof(TRecord)} is not supported."); + } + } } diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs index 91ac166aafd4..2922b4d7108c 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/GenericDataModelConformanceTests.cs @@ -45,17 +45,7 @@ await this.ExecuteAsync(async collection => Assert.Equal(inserted.Key, key); VectorStoreGenericDataModel? received = await collection.GetAsync(key, new() { IncludeVectors = true }); - Assert.NotNull(received); - - Assert.Equal(received.Key, key); - foreach (var pair in inserted.Data) - { - Assert.Equal(pair.Value, received.Data[pair.Key]); - } - - Assert.Equal( - ((ReadOnlyMemory)inserted.Vectors[EmbeddingPropertyName]!).ToArray(), - ((ReadOnlyMemory)received.Vectors[EmbeddingPropertyName]!).ToArray()); + this.AssertEqual(inserted, received, includeVectors: true); await collection.DeleteAsync(key); diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs new file mode 100644 index 000000000000..fb267239844f --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.Models; +using VectorDataSpecificationTests.Support; +using VectorDataSpecificationTests.Xunit; +using Xunit; + +namespace VectorDataSpecificationTests.CRUD; + +public class RecordConformanceTests(VectorStoreFixture fixture) + : ConformanceTestsBase>(fixture) where TKey : notnull +{ + [ConditionalFact] + public async Task ReadingAndDeletingNonExistingRecordDoesNotThrow() + { + await this.ExecuteAsync(async collection => + { + TKey key = this.Fixture.GenerateNextKey(); + + Assert.Null(await collection.GetAsync(key)); + await collection.DeleteAsync(key); + }); + } + + [ConditionalFact] + public Task CanInsertUpdateAndDelete_WithVectors() + => this.CanInsertUpdateAndDelete(includeVectors: true); + + [ConditionalFact] + public Task CanInsertUpdateAndDelete_WithoutVectors() + => this.CanInsertUpdateAndDelete(includeVectors: false); + + private async Task CanInsertUpdateAndDelete(bool includeVectors) + { + await this.ExecuteAsync(async collection => + { + TKey expectedKey = this.Fixture.GenerateNextKey(); + SimpleModel inserted = new() + { + Id = expectedKey, + Text = "some", + Number = 123, + Floats = new ReadOnlyMemory(Enumerable.Repeat(0.1f, SimpleModel.DimensionCount).ToArray()) + }; + + TKey key = await collection.UpsertAsync(inserted); + Assert.Equal(expectedKey, key); + + SimpleModel? received = await collection.GetAsync(expectedKey, new() { IncludeVectors = includeVectors }); + this.AssertEqual(inserted, received, includeVectors); + + SimpleModel updated = new() + { + Id = expectedKey, + Text = "updated", + Number = 456, + Floats = new ReadOnlyMemory(Enumerable.Repeat(0.2f, SimpleModel.DimensionCount).ToArray()) + }; + + key = await collection.UpsertAsync(updated); + Assert.Equal(expectedKey, key); + + received = await collection.GetAsync(expectedKey, new() { IncludeVectors = includeVectors }); + this.AssertEqual(updated, received, includeVectors); + + await collection.DeleteAsync(key); + + received = await collection.GetAsync(key); + Assert.Null(received); + }); + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs index 0646f0fe2f1f..3f99d6186f94 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs @@ -11,6 +11,8 @@ namespace VectorDataSpecificationTests.Models; /// TKey is a generic parameter because different connectors support different key types. public sealed class SimpleModel { + public const int DimensionCount = 10; + [VectorStoreRecordKey(StoragePropertyName = "key")] public TKey? Id { get; set; } @@ -20,6 +22,6 @@ public sealed class SimpleModel [VectorStoreRecordData(StoragePropertyName = "number")] public int Number { get; set; } - [VectorStoreRecordVector(Dimensions: 10, StoragePropertyName = "embedding")] + [VectorStoreRecordVector(Dimensions: DimensionCount, StoragePropertyName = "embedding")] public ReadOnlyMemory Floats { get; set; } }