.Net: Add integration tests for no-vector models and fix bugs

dotnet/src/Connectors/Connectors.Memory.Postgres/PostgresVectorStoreRecordPropertyMapping.cs

@@ -185,6 +185,10 @@ public static NpgsqlParameter GetNpgsqlParameter(object? value)
         {
             switch (property)
             {
+                case VectorStoreRecordKeyPropertyModel:
+                    // There is no need to create a separate index for the key property.
+                    break;
+
                 case VectorStoreRecordVectorPropertyModel vectorProperty:
                     var indexKind = vectorProperty.IndexKind ?? PostgresConstants.DefaultIndexKind;
                     var distanceFunction = vectorProperty.DistanceFunction ?? PostgresConstants.DefaultDistanceFunction;

dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordCollection.cs

@@ -118,7 +118,7 @@ public RedisJsonVectorStoreRecordCollection(IDatabase database, string collectio
         else
         {
             // Default Mapper.
-            this._mapper = new RedisJsonVectorStoreRecordMapper<TRecord>(this._model.KeyProperty, this._jsonSerializerOptions);
+            this._mapper = new RedisJsonVectorStoreRecordMapper<TRecord>(this._model, this._jsonSerializerOptions);
         }
 #pragma warning restore CS0618
     }

dotnet/src/Connectors/Connectors.Memory.Redis/RedisJsonVectorStoreRecordMapper.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Nodes;
 using Microsoft.Extensions.VectorData;
@@ -12,14 +13,14 @@ namespace Microsoft.SemanticKernel.Connectors.Redis;
 /// </summary>
 /// <typeparam name="TConsumerDataModel">The consumer data model to map to or from.</typeparam>
 internal sealed class RedisJsonVectorStoreRecordMapper<TConsumerDataModel>(
-    VectorStoreRecordKeyPropertyModel keyProperty,
+    VectorStoreRecordModel model,
     JsonSerializerOptions jsonSerializerOptions)
 #pragma warning disable CS0618 // IVectorStoreRecordMapper is obsolete
     : IVectorStoreRecordMapper<TConsumerDataModel, (string Key, JsonNode Node)>
 #pragma warning restore CS0618
 {
     /// <summary>The key property.</summary>
-    private readonly string _keyPropertyStorageName = keyProperty.StorageName;
+    private readonly string _keyPropertyStorageName = model.KeyProperty.StorageName;
 
     /// <inheritdoc />
     public (string Key, JsonNode Node) MapFromDataToStorageModel(TConsumerDataModel dataModel)
@@ -37,17 +38,24 @@ internal sealed class RedisJsonVectorStoreRecordMapper<TConsumerDataModel>(
             return (keyValue, jsonNode);
         }
 
-        throw new VectorStoreRecordMappingException($"Missing key field {this._keyPropertyStorageName} on provided record of type {typeof(TConsumerDataModel).FullName}.");
+        throw new VectorStoreRecordMappingException($"Missing key field '{this._keyPropertyStorageName}' on provided record of type {typeof(TConsumerDataModel).FullName}.");
     }
 
     /// <inheritdoc />
     public TConsumerDataModel MapFromStorageToDataModel((string Key, JsonNode Node) storageModel, StorageToDataModelMapperOptions options)
     {
-        // The redis result can be either a single object or an array with a single object in the case where we are doing an MGET.
+        // The redis result can have one of three different formats:
+        // 1. a single object
+        // 2. an array with a single object in the case where we are doing an MGET
+        // 3. a single value (string, number, etc.) in the case where there is only one property being requested because the model has only one property apart from the key
         var jsonObject = storageModel.Node switch
         {
             JsonObject topLevelJsonObject => topLevelJsonObject,
             JsonArray and [JsonObject arrayEntryJsonObject] => arrayEntryJsonObject,
+            JsonValue when model.DataProperties.Count + model.VectorProperties.Count == 1 => new JsonObject
+            {
+                [model.DataProperties.Concat<VectorStoreRecordPropertyModel>(model.VectorProperties).First().StorageName] = storageModel.Node
+            },
             _ => throw new VectorStoreRecordMappingException($"Invalid data format for document with key '{storageModel.Key}'")
         };
 

dotnet/src/Connectors/Connectors.Redis.UnitTests/RedisJsonVectorStoreRecordMapperTests.cs

@@ -20,8 +20,9 @@ public sealed class RedisJsonVectorStoreRecordMapperTests
     public void MapsAllFieldsFromDataToStorageModel()
     {
         // Arrange.
-        var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string));
-        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(keyProperty, JsonSerializerOptions.Default);
+        var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection<MultiPropsModel>.ModelBuildingOptions)
+            .Build(typeof(MultiPropsModel), null, JsonSerializerOptions.Default);
+        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(model, JsonSerializerOptions.Default);
 
         // Act.
         var actual = sut.MapFromDataToStorageModel(CreateModel("test key"));
@@ -40,8 +41,10 @@ public void MapsAllFieldsFromDataToStorageModel()
     public void MapsAllFieldsFromDataToStorageModelWithCustomSerializerOptions()
     {
         // Arrange.
-        var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string)) { StorageName = "key" };
-        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(keyProperty, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
+        var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
+        var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection<MultiPropsModel>.ModelBuildingOptions)
+            .Build(typeof(MultiPropsModel), null, jsonSerializerOptions);
+        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(model, jsonSerializerOptions);
 
         // Act.
         var actual = sut.MapFromDataToStorageModel(CreateModel("test key"));
@@ -60,8 +63,9 @@ public void MapsAllFieldsFromDataToStorageModelWithCustomSerializerOptions()
     public void MapsAllFieldsFromStorageToDataModel()
     {
         // Arrange.
-        var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string));
-        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(keyProperty, JsonSerializerOptions.Default);
+        var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection<MultiPropsModel>.ModelBuildingOptions)
+            .Build(typeof(MultiPropsModel), null, JsonSerializerOptions.Default);
+        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(model, JsonSerializerOptions.Default);
 
         // Act.
         var jsonObject = new JsonObject();
@@ -84,8 +88,10 @@ public void MapsAllFieldsFromStorageToDataModel()
     public void MapsAllFieldsFromStorageToDataModelWithCustomSerializerOptions()
     {
         // Arrange.
-        var keyProperty = new VectorStoreRecordKeyPropertyModel("Key", typeof(string)) { StorageName = "key" };
-        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(keyProperty, new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase });
+        var jsonSerializerOptions = new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
+        var model = new VectorStoreRecordJsonModelBuilder(RedisJsonVectorStoreRecordCollection<MultiPropsModel>.ModelBuildingOptions)
+            .Build(typeof(MultiPropsModel), null, jsonSerializerOptions);
+        var sut = new RedisJsonVectorStoreRecordMapper<MultiPropsModel>(model, jsonSerializerOptions);
 
         // Act.
         var jsonObject = new JsonObject();

dotnet/src/VectorDataIntegrationTests/AzureAISearchIntegrationTests/CRUD/AzureAISearchNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using AzureAISearchIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace AzureAISearchIntegrationTests.CRUD;
+
+public class AzureAISearchNoVectorConformanceTests(AzureAISearchNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<AzureAISearchNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => AzureAISearchTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CRUD/CosmosMongoDBNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using CosmosMongoDBIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace CosmosMongoDBIntegrationTests.CRUD;
+
+public class CosmosMongoDBNoVectorConformanceTests(CosmosMongoDBNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<CosmosMongoDBNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => CosmosMongoDBTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/CosmosMongoDBIntegrationTests.csproj

@@ -7,6 +7,7 @@
     <IsTestProject>true</IsTestProject>
     <IsPackable>false</IsPackable>
     <RootNamespace>CosmosMongoDBIntegrationTests</RootNamespace>
+    <UserSecretsId>b7762d10-e29b-4bb1-8b74-b6d69a667dd4</UserSecretsId>
   </PropertyGroup>
 
   <ItemGroup>
@@ -19,6 +20,7 @@
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables"/>
     <PackageReference Include="Microsoft.Extensions.Configuration.Json"/>
     <PackageReference Include="Microsoft.NET.Test.Sdk"/>
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets"/>
   </ItemGroup>
 
   <ItemGroup>

dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBSimpleModelFixture.cs

@@ -7,8 +7,4 @@ namespace CosmosMongoDBIntegrationTests.Support;
 public class CosmosMongoDBSimpleModelFixture : SimpleModelFixture<string>
 {
     public override TestStore TestStore => CosmosMongoDBTestStore.Instance;
-
-    protected override string IndexKind => Microsoft.Extensions.VectorData.IndexKind.IvfFlat;
-
-    protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance;
 }

dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestEnvironment.cs

@@ -18,6 +18,7 @@ static CosmosMongoDBTestEnvironment()
             .AddJsonFile(path: "testsettings.json", optional: true)
             .AddJsonFile(path: "testsettings.development.json", optional: true)
             .AddEnvironmentVariables()
+            .AddUserSecrets<CosmosConnectionStringRequiredAttribute>()
             .Build();
 
         ConnectionString = configuration["AzureCosmosDBMongoDB:ConnectionString"];

dotnet/src/VectorDataIntegrationTests/CosmosMongoDBIntegrationTests/Support/CosmosMongoDBTestStore.cs

@@ -21,6 +21,10 @@ public sealed class CosmosMongoDBTestStore : TestStore
     public override IVectorStore DefaultVectorStore
         => this._defaultVectorStore ?? throw new InvalidOperationException("Call InitializeAsync() first");
 
+    public override string DefaultIndexKind => Microsoft.Extensions.VectorData.IndexKind.IvfFlat;
+
+    public override string DefaultDistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance;
+
     public AzureCosmosDBMongoDBVectorStore GetVectorStore(AzureCosmosDBMongoDBVectorStoreOptions options)
         => new(this.Database, options);
 

dotnet/src/VectorDataIntegrationTests/CosmosNoSQLIntegrationTests/CRUD/CosmosNoSQLNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using CosmosNoSQLIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace CosmosNoSQLIntegrationTests.CRUD;
+
+public class CosmosNoSQLNoVectorConformanceTests(CosmosNoSQLNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<CosmosNoSQLNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => CosmosNoSqlTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/InMemoryIntegrationTests/CRUD/InMemoryNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using InMemoryIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace InMemoryIntegrationTests.CRUD;
+
+public class InMemoryNoVectorConformanceTests(InMemoryNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<InMemoryNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => InMemoryTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/MongoDBIntegrationTests/CRUD/MongoDBNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using MongoDBIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace MongoDBIntegrationTests.CRUD;
+
+public class MongoDBNoVectorConformanceTests(MongoDBNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<MongoDBNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => MongoDBTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/PostgresIntegrationTests/CRUD/PostgresNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using PostgresIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace PostgresIntegrationTests.CRUD;
+
+public class PostgresNoVectorConformanceTests(PostgresNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<PostgresNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => PostgresTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisHashSetNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using RedisIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace RedisIntegrationTests.CRUD;
+
+public class RedisHashSetNoVectorConformanceTests(RedisHashSetNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<RedisHashSetNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => RedisTestStore.HashSetInstance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/RedisIntegrationTests/CRUD/RedisJsonNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using RedisIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace RedisIntegrationTests.CRUD;
+
+public class RedisJsonNoVectorConformanceTests(RedisJsonNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<RedisJsonNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => RedisTestStore.JsonInstance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerBatchConformanceTests.cs

@@ -25,12 +25,12 @@ public Task CanSplitBatchToAccountForMaxParameterLimit_WithoutVectors()
     private async Task CanSplitBatchToAccountForMaxParameterLimit(bool includeVectors)
     {
         var collection = fixture.Collection;
-        SimpleModel<string>[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleModel<string>()
+        SimpleRecord<string>[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleRecord<string>()
         {
             Id = fixture.GenerateNextKey<string>(),
             Number = 100 + i,
             Text = i.ToString(),
-            Floats = Enumerable.Range(0, SimpleModel<string>.DimensionCount).Select(j => (float)(i + j)).ToArray()
+            Floats = Enumerable.Range(0, SimpleRecord<string>.DimensionCount).Select(j => (float)(i + j)).ToArray()
         }).ToArray();
         var keys = inserted.Select(record => record.Id).ToArray();
 
@@ -52,13 +52,13 @@ private async Task CanSplitBatchToAccountForMaxParameterLimit(bool includeVector
     public async Task UpsertBatchIsAtomic()
     {
         var collection = fixture.Collection;
-        SimpleModel<string>[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleModel<string>()
+        SimpleRecord<string>[] inserted = Enumerable.Range(0, SqlServerMaxParameters + 1).Select(i => new SimpleRecord<string>()
         {
             // The last Id is set to NULL, so it must not be inserted and the whole batch should fail
             Id = i < SqlServerMaxParameters ? fixture.GenerateNextKey<string>() : null!,
             Number = 100 + i,
             Text = i.ToString(),
-            Floats = Enumerable.Range(0, SimpleModel<string>.DimensionCount).Select(j => (float)(i + j)).ToArray()
+            Floats = Enumerable.Range(0, SimpleRecord<string>.DimensionCount).Select(j => (float)(i + j)).ToArray()
         }).ToArray();
 
         var keys = inserted.Select(record => record.Id).Where(key => key is not null).ToArray();

dotnet/src/VectorDataIntegrationTests/SqlServerIntegrationTests/CRUD/SqlServerNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using SqlServerIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace SqlServerIntegrationTests.CRUD;
+
+public class SqlServerNoVectorConformanceTests(SqlServerNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<SqlServerNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => SqlServerTestStore.Instance;
+    }
+}

dotnet/src/VectorDataIntegrationTests/SqliteIntegrationTests/CRUD/SqliteNoVectorConformanceTests.cs

@@ -0,0 +1,17 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using SqliteIntegrationTests.Support;
+using VectorDataSpecificationTests.CRUD;
+using VectorDataSpecificationTests.Support;
+using Xunit;
+
+namespace SqliteIntegrationTests.CRUD;
+
+public class SqliteNoVectorConformanceTests(SqliteNoVectorConformanceTests.Fixture fixture)
+    : NoVectorConformanceTests<string>(fixture), IClassFixture<SqliteNoVectorConformanceTests.Fixture>
+{
+    public new class Fixture : NoVectorConformanceTests<string>.Fixture
+    {
+        public override TestStore TestStore => SqliteTestStore.Instance;
+    }
+}