diff --git a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordPropertyMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordPropertyMapping.cs index 06bd10bf4773..d16383c567ba 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordPropertyMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordPropertyMapping.cs @@ -185,6 +185,10 @@ public static NpgsqlParameter GetNpgsqlParameter(object? value) { switch (property) { + case VectorStoreRecordKeyPropertyModel: + // There is no need to create a separate index for the key property. + break; + case VectorStoreRecordVectorPropertyModel vectorProperty: var indexKind = vectorProperty.IndexKind ?? PostgresConstants.DefaultIndexKind; var distanceFunction = vectorProperty.DistanceFunction ?? PostgresConstants.DefaultDistanceFunction; diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs index ef6f48e33857..6f6f9bf6c5ce 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs @@ -118,7 +118,7 @@ public RedisJsonVectorStoreRecordCollection(IDatabase database, string collectio else { // Default Mapper. - this._mapper = new RedisJsonVectorStoreRecordMapper(this._model.KeyProperty, this._jsonSerializerOptions); + this._mapper = new RedisJsonVectorStoreRecordMapper(this._model, this._jsonSerializerOptions); } #pragma warning restore CS0618 } diff --git a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordMapper.cs index b128be837bfb..64a0ce9e5b76 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordMapper.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Linq; using System.Text.Json; using System.Text.Json.Nodes; using Microsoft.Extensions.VectorData; @@ -12,14 +13,14 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; /// /// The consumer data model to map to or from. internal sealed class RedisJsonVectorStoreRecordMapper( - VectorStoreRecordKeyPropertyModel keyProperty, + VectorStoreRecordModel model, JsonSerializerOptions jsonSerializerOptions) #pragma warning disable CS0618 // IVectorStoreRecordMapper is obsolete : IVectorStoreRecordMapper #pragma warning restore CS0618 { /// The key property. - private readonly string _keyPropertyStorageName = keyProperty.StorageName; + private readonly string _keyPropertyStorageName = model.KeyProperty.StorageName; /// public (string Key, JsonNode Node) MapFromDataToStorageModel(TConsumerDataModel dataModel) @@ -37,17 +38,24 @@ internal sealed class RedisJsonVectorStoreRecordMapper( return (keyValue, jsonNode); } - throw new VectorStoreRecordMappingException($"Missing key field {this._keyPropertyStorageName} on provided record of type {typeof(TConsumerDataModel).FullName}."); + throw new VectorStoreRecordMappingException($"Missing key field '{this._keyPropertyStorageName}' on provided record of type {typeof(TConsumerDataModel).FullName}."); } /// public TConsumerDataModel MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, StorageToDataModelMapperOptions options) { - // The redis result can be either a single object or an array with a single object in the case where we are doing an MGET. + // The redis result can have one of three different formats: + // 1. a single object + // 2. an array with a single object in the case where we are doing an MGET + // 3. a single value (string, number, etc.) in the case where there is only one property being requested because the model has only one property apart from the key var jsonObject = storageModel.Node switch { JsonObject topLevelJsonObject => topLevelJsonObject, JsonArray and [JsonObject arrayEntryJsonObject] => arrayEntryJsonObject, + JsonValue when model.DataProperties.Count + model.VectorProperties.Count == 1 => new JsonObject + { + [model.DataProperties.Concat(model.VectorProperties).First().StorageName] = storageModel.Node + }, _ => throw new VectorStoreRecordMappingException($"Invalid data format for document with key '{storageModel.Key}'") }; diff --git a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordMapperTests.cs index c3e8eb077b39..bb1b0889bd6e 100644 --- a/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordMapperTests.cs @@ -20,8 +20,9 @@ public sealed class RedisJsonVectorStoreRecordMapperTests public void MapsAllFieldsFromDataToStorageModel() { // Arrange. - var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string)); - var sut = new RedisJsonVectorStoreRecordMapper(keyProperty, JsonSerializerOptions.Default); + var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), null, JsonSerializerOptions.Default); + var sut = new RedisJsonVectorStoreRecordMapper(model, JsonSerializerOptions.Default); // Act. var actual = sut.MapFromDataToStorageModel(CreateModel("test key")); @@ -40,8 +41,10 @@ public void MapsAllFieldsFromDataToStorageModel() public void MapsAllFieldsFromDataToStorageModelWithCustomSerializerOptions() { // Arrange. - var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string)) { StorageName = "key" }; - var sut = new RedisJsonVectorStoreRecordMapper(keyProperty, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), null, jsonSerializerOptions); + var sut = new RedisJsonVectorStoreRecordMapper(model, jsonSerializerOptions); // Act. var actual = sut.MapFromDataToStorageModel(CreateModel("test key")); @@ -60,8 +63,9 @@ public void MapsAllFieldsFromDataToStorageModelWithCustomSerializerOptions() public void MapsAllFieldsFromStorageToDataModel() { // Arrange. - var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string)); - var sut = new RedisJsonVectorStoreRecordMapper(keyProperty, JsonSerializerOptions.Default); + var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), null, JsonSerializerOptions.Default); + var sut = new RedisJsonVectorStoreRecordMapper(model, JsonSerializerOptions.Default); // Act. var jsonObject = new JsonObject(); @@ -84,8 +88,10 @@ public void MapsAllFieldsFromStorageToDataModel() public void MapsAllFieldsFromStorageToDataModelWithCustomSerializerOptions() { // Arrange. - var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string)) { StorageName = "key" }; - var sut = new RedisJsonVectorStoreRecordMapper(keyProperty, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); + var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; + var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection.ModelBuildingOptions) + .Build(typeof(MultiPropsModel), null, jsonSerializerOptions); + var sut = new RedisJsonVectorStoreRecordMapper(model, jsonSerializerOptions); // Act. var jsonObject = new JsonObject(); diff --git a/dotnet/src/VectorDataIntegrationTests/AzureAISearchIntegrationTests/CRUD/AzureAISearchNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/AzureAISearchIntegrationTests/CRUD/AzureAISearchNoVectorConformanceTests.cs new file mode 100644 index 000000000000..9d16e29ed86c --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/AzureAISearchIntegrationTests/CRUD/AzureAISearchNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using AzureAISearchIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace AzureAISearchIntegrationTests.CRUD; + +public class AzureAISearchNoVectorConformanceTests(AzureAISearchNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => AzureAISearchTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CRUD/CosmosMongoDBNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CRUD/CosmosMongoDBNoVectorConformanceTests.cs new file mode 100644 index 000000000000..134707f35575 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CRUD/CosmosMongoDBNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using CosmosMongoDBIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace CosmosMongoDBIntegrationTests.CRUD; + +public class CosmosMongoDBNoVectorConformanceTests(CosmosMongoDBNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => CosmosMongoDBTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CosmosMongoDBIntegrationTests.csproj b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CosmosMongoDBIntegrationTests.csproj index 59a720d7dddd..0e4200084d4e 100644 --- a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CosmosMongoDBIntegrationTests.csproj +++ b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CosmosMongoDBIntegrationTests.csproj @@ -7,6 +7,7 @@ true false CosmosMongoDBIntegrationTests + b7762d10-e29b-4bb1-8b74-b6d69a667dd4 @@ -19,6 +20,7 @@ + diff --git a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBSimpleModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBSimpleModelFixture.cs index 90c0e3efc659..42d6a8dbf3a9 100644 --- a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBSimpleModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBSimpleModelFixture.cs @@ -7,8 +7,4 @@ namespace CosmosMongoDBIntegrationTests.Support; public class CosmosMongoDBSimpleModelFixture : SimpleModelFixture { public override TestStore TestStore => CosmosMongoDBTestStore.Instance; - - protected override string IndexKind => Microsoft.Extensions.VectorData.IndexKind.IvfFlat; - - protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; } diff --git a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestEnvironment.cs b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestEnvironment.cs index df6550d05237..faf467122f2f 100644 --- a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestEnvironment.cs +++ b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestEnvironment.cs @@ -18,6 +18,7 @@ static CosmosMongoDBTestEnvironment() .AddJsonFile(path: "testsettings.json", optional: true) .AddJsonFile(path: "testsettings.development.json", optional: true) .AddEnvironmentVariables() + .AddUserSecrets() .Build(); ConnectionString = configuration["AzureCosmosDBMongoDB:ConnectionString"]; diff --git a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestStore.cs b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestStore.cs index fba1d18c8a7f..f7727ee324fb 100644 --- a/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestStore.cs +++ b/dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestStore.cs @@ -21,6 +21,10 @@ public sealed class CosmosMongoDBTestStore : TestStore public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Call InitializeAsync() first"); + public override string DefaultIndexKind => Microsoft.Extensions.VectorData.IndexKind.IvfFlat; + + public override string DefaultDistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + public AzureCosmosDBMongoDBVectorStore GetVectorStore(AzureCosmosDBMongoDBVectorStoreOptions options) => new(this.Database, options); diff --git a/dotnet/src/VectorDataIntegrationTests/CosmosNoSQLIntegrationTests/CRUD/CosmosNoSQLNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/CosmosNoSQLIntegrationTests/CRUD/CosmosNoSQLNoVectorConformanceTests.cs new file mode 100644 index 000000000000..29f2b80ac866 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/CosmosNoSQLIntegrationTests/CRUD/CosmosNoSQLNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using CosmosNoSQLIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace CosmosNoSQLIntegrationTests.CRUD; + +public class CosmosNoSQLNoVectorConformanceTests(CosmosNoSQLNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => CosmosNoSqlTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/InMemoryIntegrationTests/CRUD/InMemoryNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/InMemoryIntegrationTests/CRUD/InMemoryNoVectorConformanceTests.cs new file mode 100644 index 000000000000..7c8c759c8d3c --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/InMemoryIntegrationTests/CRUD/InMemoryNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using InMemoryIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace InMemoryIntegrationTests.CRUD; + +public class InMemoryNoVectorConformanceTests(InMemoryNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => InMemoryTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/MongoDBIntegrationTests/CRUD/MongoDBNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/MongoDBIntegrationTests/CRUD/MongoDBNoVectorConformanceTests.cs new file mode 100644 index 000000000000..f4a597f6bcaa --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/MongoDBIntegrationTests/CRUD/MongoDBNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using MongoDBIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace MongoDBIntegrationTests.CRUD; + +public class MongoDBNoVectorConformanceTests(MongoDBNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => MongoDBTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresNoVectorConformanceTests.cs new file mode 100644 index 000000000000..93f6a0fff133 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using PostgresIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace PostgresIntegrationTests.CRUD; + +public class PostgresNoVectorConformanceTests(PostgresNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => PostgresTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisHashSetNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisHashSetNoVectorConformanceTests.cs new file mode 100644 index 000000000000..dddd33af25e9 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisHashSetNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using RedisIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace RedisIntegrationTests.CRUD; + +public class RedisHashSetNoVectorConformanceTests(RedisHashSetNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => RedisTestStore.HashSetInstance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisJsonNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisJsonNoVectorConformanceTests.cs new file mode 100644 index 000000000000..3d3828244bd9 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisJsonNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using RedisIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace RedisIntegrationTests.CRUD; + +public class RedisJsonNoVectorConformanceTests(RedisJsonNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => RedisTestStore.JsonInstance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs index b9e6f54eb752..01dacbeeb6f8 100644 --- a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs @@ -25,12 +25,12 @@ public Task CanSplitBatchToAccountForMaxParameterLimit_WithoutVectors() private async Task CanSplitBatchToAccountForMaxParameterLimit(bool includeVectors) { var collection = fixture.Collection; - SimpleModel[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleModel() + SimpleRecord[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleRecord() { Id = fixture.GenerateNextKey(), Number = 100 + i, Text = i.ToString(), - Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + Floats = Enumerable.Range(0, SimpleRecord.DimensionCount).Select(j => (float)(i + j)).ToArray() }).ToArray(); var keys = inserted.Select(record => record.Id).ToArray(); @@ -52,13 +52,13 @@ private async Task CanSplitBatchToAccountForMaxParameterLimit(bool includeVector public async Task UpsertBatchIsAtomic() { var collection = fixture.Collection; - SimpleModel[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleModel() + SimpleRecord[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleRecord() { // The last Id is set to NULL, so it must not be inserted and the whole batch should fail Id = i < SqlServerMaxParameters ? fixture.GenerateNextKey() : null!, Number = 100 + i, Text = i.ToString(), - Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + Floats = Enumerable.Range(0, SimpleRecord.DimensionCount).Select(j => (float)(i + j)).ToArray() }).ToArray(); var keys = inserted.Select(record => record.Id).Where(key => key is not null).ToArray(); diff --git a/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerNoVectorConformanceTests.cs new file mode 100644 index 000000000000..7b70e75d7b70 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using SqlServerIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace SqlServerIntegrationTests.CRUD; + +public class SqlServerNoVectorConformanceTests(SqlServerNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => SqlServerTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/SqliteIntegrationTests/CRUD/SqliteNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/SqliteIntegrationTests/CRUD/SqliteNoVectorConformanceTests.cs new file mode 100644 index 000000000000..1b0e800e8ca7 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/SqliteIntegrationTests/CRUD/SqliteNoVectorConformanceTests.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. + +using SqliteIntegrationTests.Support; +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using Xunit; + +namespace SqliteIntegrationTests.CRUD; + +public class SqliteNoVectorConformanceTests(SqliteNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => SqliteTestStore.Instance; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs index 849789b5e910..ac50caf47040 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/BatchConformanceTests.cs @@ -67,12 +67,12 @@ public Task UpsertBatchAsyncCanInsertNewRecord_WithoutVectors() private async Task UpsertBatchAsyncCanInsertNewRecords(bool includeVectors) { var collection = fixture.Collection; - SimpleModel[] inserted = Enumerable.Range(0, 10).Select(i => new SimpleModel() + SimpleRecord[] inserted = Enumerable.Range(0, 10).Select(i => new SimpleRecord() { Id = fixture.GenerateNextKey(), Number = 100 + i, Text = i.ToString(), - Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + Floats = Enumerable.Range(0, SimpleRecord.DimensionCount).Select(j => (float)(i + j)).ToArray() }).ToArray(); var keys = inserted.Select(record => record.Id).ToArray(); @@ -97,16 +97,16 @@ public Task UpsertBatchAsyncCanUpdateExistingRecords_WithoutVectors() private async Task UpsertBatchAsyncCanUpdateExistingRecords(bool includeVectors) { - SimpleModel[] inserted = Enumerable.Range(0, 10).Select(i => new SimpleModel() + SimpleRecord[] inserted = Enumerable.Range(0, 10).Select(i => new SimpleRecord() { Id = fixture.GenerateNextKey(), Number = 100 + i, Text = i.ToString(), - Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + Floats = Enumerable.Range(0, SimpleRecord.DimensionCount).Select(j => (float)(i + j)).ToArray() }).ToArray(); await fixture.Collection.UpsertAsync(inserted).ToArrayAsync(); - SimpleModel[] updated = inserted.Select(i => new SimpleModel() + SimpleRecord[] updated = inserted.Select(i => new SimpleRecord() { Id = i.Id, Text = i.Text + "updated", @@ -136,16 +136,16 @@ public Task UpsertCanBothInsertAndUpdateRecordsFromTheSameBatch_WithoutVectors() private async Task UpsertCanBothInsertAndUpdateRecordsFromTheSameBatch(bool includeVectors) { - SimpleModel[] records = Enumerable.Range(0, 10).Select(i => new SimpleModel() + SimpleRecord[] records = Enumerable.Range(0, 10).Select(i => new SimpleRecord() { Id = fixture.GenerateNextKey(), Number = 100 + i, Text = i.ToString(), - Floats = Enumerable.Range(0, SimpleModel.DimensionCount).Select(j => (float)(i + j)).ToArray() + Floats = Enumerable.Range(0, SimpleRecord.DimensionCount).Select(j => (float)(i + j)).ToArray() }).ToArray(); // We take first half of the records and insert them. - SimpleModel[] firstHalf = records.Take(records.Length / 2).ToArray(); + SimpleRecord[] firstHalf = records.Take(records.Length / 2).ToArray(); TKey[] insertedKeys = await fixture.Collection.UpsertAsync(firstHalf).ToArrayAsync(); Assert.Equal( firstHalf.Select(r => r.Id).OrderBy(id => id).ToArray(), @@ -196,6 +196,6 @@ public async Task DeleteBatchAsyncDeletesTheRecords() // The order of records in the received array is not guaranteed // to match the order of keys in the requested keys array. - protected SimpleModel GetRecord(SimpleModel[] received, TKey key) + protected SimpleRecord GetRecord(SimpleRecord[] received, TKey key) => received.Single(r => r.Id!.Equals(key)); } diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/NoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/NoVectorConformanceTests.cs new file mode 100644 index 000000000000..d5c468f07cb9 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/NoVectorConformanceTests.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.VectorData; +using VectorDataSpecificationTests.Support; +using VectorDataSpecificationTests.Xunit; +using Xunit; + +namespace VectorDataSpecificationTests.CRUD; + +/// +/// Tests CRUD operations using a model without a vector. +/// This is only supported by a subset of databases so only extend if applicable for your database. +/// +public class NoVectorConformanceTests(NoVectorConformanceTests.Fixture fixture) where TKey : notnull +{ + [ConditionalFact] + public Task GetAsyncReturnsInsertedRecord_WithVectors() + => this.GetAsyncReturnsInsertedRecord(includeVectors: true); + + [ConditionalFact] + public Task GetAsyncReturnsInsertedRecord_WithoutVectors() + => this.GetAsyncReturnsInsertedRecord(includeVectors: false); + + private async Task GetAsyncReturnsInsertedRecord(bool includeVectors) + { + var expectedRecord = fixture.TestData[0]; + + var received = await fixture.Collection.GetAsync(expectedRecord.Id, new() { IncludeVectors = includeVectors }); + + expectedRecord.AssertEqual(received); + } + + [ConditionalFact] + public Task UpsertAsyncCanInsertNewRecord_WithVectors() + => this.UpsertAsyncCanInsertNewRecord(includeVectors: true); + + [ConditionalFact] + public Task UpsertAsyncCanInsertNewRecord_WithoutVectors() + => this.UpsertAsyncCanInsertNewRecord(includeVectors: false); + + private async Task UpsertAsyncCanInsertNewRecord(bool includeVectors) + { + var collection = fixture.Collection; + TKey expectedKey = fixture.GenerateNextKey(); + NoVectorRecord inserted = new() + { + Id = expectedKey, + Text = "some" + }; + + Assert.Null(await collection.GetAsync(expectedKey)); + TKey key = await collection.UpsertAsync(inserted); + Assert.Equal(expectedKey, key); + + var received = await collection.GetAsync(expectedKey, new() { IncludeVectors = includeVectors }); + inserted.AssertEqual(received); + } + + [ConditionalFact] + public Task UpsertAsyncCanUpdateExistingRecord_WithVectors() + => this.UpsertAsyncCanUpdateExistingRecord(includeVectors: true); + + [ConditionalFact] + public Task UpsertAsyncCanUpdateExistingRecord__WithoutVectors() + => this.UpsertAsyncCanUpdateExistingRecord(includeVectors: false); + + private async Task UpsertAsyncCanUpdateExistingRecord(bool includeVectors) + { + var collection = fixture.Collection; + var existingRecord = fixture.TestData[1]; + NoVectorRecord updated = new() + { + Id = existingRecord.Id, + Text = "updated" + }; + + Assert.NotNull(await collection.GetAsync(existingRecord.Id)); + TKey key = await collection.UpsertAsync(updated); + Assert.Equal(existingRecord.Id, key); + + var received = await collection.GetAsync(existingRecord.Id, new() { IncludeVectors = includeVectors }); + updated.AssertEqual(received); + } + + [ConditionalFact] + public async Task DeleteAsyncDeletesTheRecord() + { + var recordToRemove = fixture.TestData[2]; + + Assert.NotNull(await fixture.Collection.GetAsync(recordToRemove.Id)); + await fixture.Collection.DeleteAsync(recordToRemove.Id); + Assert.Null(await fixture.Collection.GetAsync(recordToRemove.Id)); + } + + /// + /// This class is for testing databases that support having no vector. + /// Not all DBs support this. + /// + public sealed class NoVectorRecord + { + public const int DimensionCount = 3; + + [VectorStoreRecordKey(StoragePropertyName = "key")] + public TKey Id { get; set; } = default!; + + [VectorStoreRecordData(StoragePropertyName = "text")] + public string? Text { get; set; } + + public void AssertEqual(NoVectorRecord? other) + { + Assert.NotNull(other); + Assert.Equal(this.Id, other.Id); + Assert.Equal(this.Text, other.Text); + } + } + + /// + /// Provides data and configuration for a model without a vector, which is supported by some connectors. + /// + public abstract class Fixture : VectorStoreCollectionFixture + { + protected override List BuildTestData() => + [ + new() + { + Id = this.GenerateNextKey(), + Text = "UsedByGetTests", + }, + new() + { + Id = this.GenerateNextKey(), + Text = "UsedByUpdateTests", + }, + new() + { + Id = this.GenerateNextKey(), + Text = "UsedByDeleteTests", + }, + new() + { + Id = this.GenerateNextKey(), + Text = "UsedByDeleteBatchTests", + } + ]; + + protected override VectorStoreRecordDefinition GetRecordDefinition() + => new() + { + Properties = + [ + new VectorStoreRecordKeyProperty(nameof(NoVectorRecord.Id), typeof(TKey)), + new VectorStoreRecordDataProperty(nameof(NoVectorRecord.Text), typeof(string)) { IsFilterable = true }, + ] + }; + + protected override async Task WaitForDataAsync() + { + for (var i = 0; i < 20; i++) + { + var results = await this.Collection.GetAsync([this.TestData[0].Id, this.TestData[1].Id, this.TestData[2].Id, this.TestData[3].Id]).ToArrayAsync(); + if (results.Length == 4 && results.All(r => r != null)) + { + return; + } + + await Task.Delay(TimeSpan.FromMilliseconds(100)); + } + + throw new InvalidOperationException("Data did not appear in the collection within the expected time."); + } + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs index c2a63b86cae8..a13f19696d55 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/CRUD/RecordConformanceTests.cs @@ -53,12 +53,12 @@ private async Task UpsertAsyncCanInsertNewRecord(bool includeVectors) { var collection = fixture.Collection; TKey expectedKey = fixture.GenerateNextKey(); - SimpleModel inserted = new() + SimpleRecord inserted = new() { Id = expectedKey, Text = "some", Number = 123, - Floats = new ReadOnlyMemory(Enumerable.Repeat(0.1f, SimpleModel.DimensionCount).ToArray()) + Floats = new ReadOnlyMemory(Enumerable.Repeat(0.1f, SimpleRecord.DimensionCount).ToArray()) }; Assert.Null(await collection.GetAsync(expectedKey)); @@ -81,12 +81,12 @@ private async Task UpsertAsyncCanUpdateExistingRecord(bool includeVectors) { var collection = fixture.Collection; var existingRecord = fixture.TestData[1]; - SimpleModel updated = new() + SimpleRecord updated = new() { Id = existingRecord.Id, Text = "updated", Number = 456, - Floats = new ReadOnlyMemory(Enumerable.Repeat(0.2f, SimpleModel.DimensionCount).ToArray()) + Floats = new ReadOnlyMemory(Enumerable.Repeat(0.2f, SimpleRecord.DimensionCount).ToArray()) }; Assert.NotNull(await collection.GetAsync(existingRecord.Id)); diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 16f8679df842..a3d67bb4410b 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -12,7 +12,7 @@ public abstract class CollectionConformanceTests(VectorStoreFixture fixtur { [ConditionalFact] public Task DeleteCollectionDoesNotThrowForNonExistingCollection() - => this.DeleteNonExistingCollection>(); + => this.DeleteNonExistingCollection>(); [ConditionalFact] public Task DeleteCollectionDoesNotThrowForNonExistingCollection_GenericDataModel() @@ -20,7 +20,7 @@ public Task DeleteCollectionDoesNotThrowForNonExistingCollection_GenericDataMode [ConditionalFact] public Task CreateCollectionCreatesTheCollection() - => this.CreateCollection>(); + => this.CreateCollection>(); [ConditionalFact] public Task CreateCollectionCreatesTheCollection_GenericDataModel() @@ -28,7 +28,7 @@ public Task CreateCollectionCreatesTheCollection_GenericDataModel() [ConditionalFact] public Task CreateCollectionIfNotExistsCalledMoreThanOnceDoesNotThrow() - => this.CreateCollectionIfNotExistsMoreThanOnce>(); + => this.CreateCollectionIfNotExistsMoreThanOnce>(); [ConditionalFact] public Task CreateCollectionIfNotExistsCalledMoreThanOnceDoesNotThrow_GenericDataModel() @@ -36,7 +36,7 @@ public Task CreateCollectionIfNotExistsCalledMoreThanOnceDoesNotThrow_GenericDat [ConditionalFact] public Task CreateCollectionCalledMoreThanOnceThrowsVectorStoreOperationException() - => this.CreateCollectionMoreThanOnce>(); + => this.CreateCollectionMoreThanOnce>(); [ConditionalFact] public Task CreateCollectionCalledMoreThanOnceThrowsVectorStoreOperationException_GenericDataModel() diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleRecord.cs similarity index 91% rename from dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs rename to dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleRecord.cs index 13a47e386516..1d2222204045 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleModel.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Models/SimpleRecord.cs @@ -10,7 +10,7 @@ namespace VectorDataSpecificationTests.Models; /// a key, int, string and an embedding. /// /// TKey is a generic parameter because different connectors support different key types. -public sealed class SimpleModel +public sealed class SimpleRecord { public const int DimensionCount = 3; @@ -26,7 +26,7 @@ public sealed class SimpleModel [VectorStoreRecordVector(Dimensions: DimensionCount, StoragePropertyName = "embedding")] public ReadOnlyMemory Floats { get; set; } - public void AssertEqual(SimpleModel? other, bool includeVectors) + public void AssertEqual(SimpleRecord? other, bool includeVectors) { Assert.NotNull(other); Assert.Equal(this.Id, other.Id); diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Support/SimpleModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Support/SimpleModelFixture.cs index b5c688c01835..370a77fae976 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Support/SimpleModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Support/SimpleModelFixture.cs @@ -5,38 +5,38 @@ namespace VectorDataSpecificationTests.Support; -public abstract class SimpleModelFixture : VectorStoreCollectionFixture> +public abstract class SimpleModelFixture : VectorStoreCollectionFixture> where TKey : notnull { - protected override List> BuildTestData() => + protected override List> BuildTestData() => [ new() { Id = this.GenerateNextKey(), Number = 1, Text = "UsedByGetTests", - Floats = Enumerable.Repeat(0.1f, SimpleModel.DimensionCount).ToArray() + Floats = Enumerable.Repeat(0.1f, SimpleRecord.DimensionCount).ToArray() }, new() { Id = this.GenerateNextKey(), Number = 2, Text = "UsedByUpdateTests", - Floats = Enumerable.Repeat(0.2f, SimpleModel.DimensionCount).ToArray() + Floats = Enumerable.Repeat(0.2f, SimpleRecord.DimensionCount).ToArray() }, new() { Id = this.GenerateNextKey(), Number = 3, Text = "UsedByDeleteTests", - Floats = Enumerable.Repeat(0.3f, SimpleModel.DimensionCount).ToArray() + Floats = Enumerable.Repeat(0.3f, SimpleRecord.DimensionCount).ToArray() }, new() { Id = this.GenerateNextKey(), Number = 4, Text = "UsedByDeleteBatchTests", - Floats = Enumerable.Repeat(0.4f, SimpleModel.DimensionCount).ToArray() + Floats = Enumerable.Repeat(0.4f, SimpleRecord.DimensionCount).ToArray() } ]; @@ -45,16 +45,16 @@ protected override VectorStoreRecordDefinition GetRecordDefinition() { Properties = [ - new VectorStoreRecordKeyProperty(nameof(SimpleModel.Id), typeof(TKey)), - new VectorStoreRecordVectorProperty(nameof(SimpleModel.Floats), typeof(ReadOnlyMemory?)) + new VectorStoreRecordKeyProperty(nameof(SimpleRecord.Id), typeof(TKey)), + new VectorStoreRecordVectorProperty(nameof(SimpleRecord.Floats), typeof(ReadOnlyMemory?)) { - Dimensions = SimpleModel.DimensionCount, + Dimensions = SimpleRecord.DimensionCount, DistanceFunction = this.DistanceFunction, IndexKind = this.IndexKind }, - new VectorStoreRecordDataProperty(nameof(SimpleModel.Number), typeof(int)) { IsFilterable = true }, - new VectorStoreRecordDataProperty(nameof(SimpleModel.Text), typeof(string)) { IsFilterable = true }, + new VectorStoreRecordDataProperty(nameof(SimpleRecord.Number), typeof(int)) { IsFilterable = true }, + new VectorStoreRecordDataProperty(nameof(SimpleRecord.Text), typeof(string)) { IsFilterable = true }, ] }; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs new file mode 100644 index 000000000000..016ab64870e6 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.CRUD; +using VectorDataSpecificationTests.Support; +using WeaviateIntegrationTests.Support; +using Xunit; + +namespace WeaviateIntegrationTests.CRUD; + +public class WeaviateNoVectorConformanceTests(WeaviateNoVectorConformanceTests.Fixture fixture) + : NoVectorConformanceTests(fixture), IClassFixture +{ + public new class Fixture : NoVectorConformanceTests.Fixture + { + public override TestStore TestStore => WeaviateTestStore.Instance; + + /// + /// Weaviate collections must start with an uppercase letter. + /// + protected override string CollectionName => "NoVectorCollection"; + } +}