From a52826f41448fd5c549c859c7b7096211a0ad0c5 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 13:43:09 -0700 Subject: [PATCH 01/15] Added support for single vector data schema in Weaviate --- .../WeaviateConstants.cs | 3 + .../WeaviateModelBuilder.cs | 27 ++++--- .../WeaviateVectorStoreRecordCollection.cs | 9 ++- ...viateVectorStoreRecordCollectionOptions.cs | 8 ++ .../WeaviateVectorStoreRecordMapper.cs | 49 ++++++++++-- .../WeaviateGenericDataModelMapperTests.cs | 8 +- ...VectorStoreCollectionCreateMappingTests.cs | 12 +-- ...rStoreRecordCollectionQueryBuilderTests.cs | 2 +- .../WeaviateVectorStoreRecordMapperTests.cs | 77 ++++++++++++------- 9 files changed, 139 insertions(+), 56 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs index f98b4ec35fde..bcb4d1f53b56 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs @@ -16,6 +16,9 @@ internal sealed class WeaviateConstants /// Reserved vector property name in Weaviate. internal const string ReservedVectorPropertyName = "vectors"; + /// Reserved single vector property name in Weaviate. + internal const string ReservedSingleVectorPropertyName = "vector"; + /// Collection property name in Weaviate. internal const string CollectionPropertyName = "class"; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs index eaac5cc83f44..967c9cd5738c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs @@ -6,22 +6,25 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; -internal class WeaviateModelBuilder() : VectorStoreRecordJsonModelBuilder(s_modelBuildingOptions) +internal class WeaviateModelBuilder(bool useSingleVector) : VectorStoreRecordJsonModelBuilder(GetModelBuildingOptions(useSingleVector)) { - private static readonly VectorStoreRecordModelBuildingOptions s_modelBuildingOptions = new() + private static VectorStoreRecordModelBuildingOptions GetModelBuildingOptions(bool useSingleVector) { - RequiresAtLeastOneVector = false, - SupportsMultipleKeys = false, - SupportsMultipleVectors = true, + return new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleKeys = false, + SupportsMultipleVectors = !useSingleVector, - SupportedKeyPropertyTypes = [typeof(Guid)], - SupportedDataPropertyTypes = s_supportedDataTypes, - SupportedEnumerableDataPropertyElementTypes = s_supportedDataTypes, - SupportedVectorPropertyTypes = s_supportedVectorTypes, + SupportedKeyPropertyTypes = [typeof(Guid)], + SupportedDataPropertyTypes = s_supportedDataTypes, + SupportedEnumerableDataPropertyElementTypes = s_supportedDataTypes, + SupportedVectorPropertyTypes = s_supportedVectorTypes, - UsesExternalSerializer = true, - ReservedKeyStorageName = WeaviateConstants.ReservedKeyPropertyName - }; + UsesExternalSerializer = true, + ReservedKeyStorageName = WeaviateConstants.ReservedKeyPropertyName + }; + } private static readonly HashSet s_supportedDataTypes = [ diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 400c0d9f721b..9d871a5976f8 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -95,7 +95,8 @@ public WeaviateVectorStoreRecordCollection( this.CollectionName = collectionName; this._options = options ?? new(); this._apiKey = this._options.ApiKey; - this._model = new WeaviateModelBuilder().Build(typeof(TRecord), this._options.VectorStoreRecordDefinition, s_jsonSerializerOptions); + this._model = new WeaviateModelBuilder(this._options.UseSingleVector) + .Build(typeof(TRecord), this._options.VectorStoreRecordDefinition, s_jsonSerializerOptions); // Assign mapper. this._mapper = this.InitializeMapper(); @@ -467,7 +468,11 @@ private IVectorStoreRecordMapper InitializeMapper() return (mapper as IVectorStoreRecordMapper)!; } - return new WeaviateVectorStoreRecordMapper(this.CollectionName, this._model, s_jsonSerializerOptions); + return new WeaviateVectorStoreRecordMapper( + this.CollectionName, + this._options.UseSingleVector, + this._model, + s_jsonSerializerOptions); } #pragma warning restore CS0618 diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs index 7dff47f28d0e..7ef5b570387a 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs @@ -39,4 +39,12 @@ public sealed class WeaviateVectorStoreRecordCollectionOptions /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. /// public string? ApiKey { get; set; } = null; + + /// + /// Boolean flag which indicates whether a single vector data schema should be used in Weaviate collection. + /// By default it's , meaning that multiple vector data schema will be used (also known as named vectors). + /// . + /// When single vector data schema is enabled, only one vector property will be used with name "vector". + /// + public bool UseSingleVector { get; set; } = false; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs index 93edfdaadd96..f148ea2eb9b1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs @@ -13,17 +13,26 @@ internal sealed class WeaviateVectorStoreRecordMapper : IVectorStoreRec #pragma warning restore CS0618 { private readonly string _collectionName; + private readonly bool _useSingleVector; private readonly VectorStoreRecordModel _model; private readonly JsonSerializerOptions _jsonSerializerOptions; + private readonly string _vectorPropertyName; + public WeaviateVectorStoreRecordMapper( string collectionName, + bool useSingleVector, VectorStoreRecordModel model, JsonSerializerOptions jsonSerializerOptions) { this._collectionName = collectionName; + this._useSingleVector = useSingleVector; this._model = model; this._jsonSerializerOptions = jsonSerializerOptions; + + this._vectorPropertyName = useSingleVector ? + WeaviateConstants.ReservedSingleVectorPropertyName : + WeaviateConstants.ReservedVectorPropertyName; } public JsonObject MapFromDataToStorageModel(TRecord dataModel) @@ -41,7 +50,7 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel) // account e.g. naming policies. TemporaryStorageName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set { WeaviateConstants.ReservedKeyPropertyName, jsonNodeDataModel[this._model.KeyProperty.TemporaryStorageName!]!.DeepClone() }, { WeaviateConstants.ReservedDataPropertyName, new JsonObject() }, - { WeaviateConstants.ReservedVectorPropertyName, new JsonObject() }, + { this._vectorPropertyName, new JsonObject() }, }; // Populate data properties. @@ -56,13 +65,26 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel) } // Populate vector properties. - foreach (var property in this._model.VectorProperties) + if (this._useSingleVector) { - var node = jsonNodeDataModel[property.StorageName]; + var vectorProperty = this._model.VectorProperty; + var node = jsonNodeDataModel[vectorProperty.StorageName]; if (node is not null) { - weaviateObjectModel[WeaviateConstants.ReservedVectorPropertyName]![property.StorageName] = node.DeepClone(); + weaviateObjectModel[this._vectorPropertyName] = node.DeepClone(); + } + } + else + { + foreach (var property in this._model.VectorProperties) + { + var node = jsonNodeDataModel[property.StorageName]; + + if (node is not null) + { + weaviateObjectModel[this._vectorPropertyName]![property.StorageName] = node.DeepClone(); + } } } @@ -97,13 +119,26 @@ public TRecord MapFromStorageToDataModel(JsonObject storageModel, StorageToDataM // Populate vector properties. if (options.IncludeVectors) { - foreach (var property in this._model.VectorProperties) + if (this._useSingleVector) { - var node = storageModel[WeaviateConstants.ReservedVectorPropertyName]?[property.StorageName]; + var vectorProperty = this._model.VectorProperty; + var node = storageModel[this._vectorPropertyName]; if (node is not null) { - jsonNodeDataModel[property.StorageName] = node.DeepClone(); + jsonNodeDataModel[vectorProperty.StorageName] = node.DeepClone(); + } + } + else + { + foreach (var property in this._model.VectorProperties) + { + var node = storageModel[this._vectorPropertyName]?[property.StorageName]; + + if (node is not null) + { + jsonNodeDataModel[property.StorageName] = node.DeepClone(); + } } } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs index 15193f410b4f..50b70b38ad4d 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs @@ -18,6 +18,8 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateGenericDataModelMapperTests { + private const bool UseSingleVector = false; + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -29,7 +31,7 @@ public sealed class WeaviateGenericDataModelMapperTests } }; - private static readonly VectorStoreRecordModel s_model = new WeaviateModelBuilder() + private static readonly VectorStoreRecordModel s_model = new WeaviateModelBuilder(UseSingleVector) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -351,7 +353,7 @@ public void MapFromDataToStorageModelSkipsMissingProperties() ] }; - var model = new WeaviateModelBuilder().Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); + var model = new WeaviateModelBuilder(UseSingleVector).Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); var key = new Guid("55555555-5555-5555-5555-555555555555"); @@ -382,7 +384,7 @@ public void MapFromStorageToDataModelSkipsMissingProperties() ] }; - var model = new WeaviateModelBuilder().Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); + var model = new WeaviateModelBuilder(UseSingleVector).Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); var key = new Guid("55555555-5555-5555-5555-555555555555"); diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs index 0203e726c145..6b3be9496b97 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -14,11 +14,13 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateVectorStoreCollectionCreateMappingTests { + private const bool UseSingleVector = false; + [Fact] public void ItThrowsExceptionWithInvalidIndexKind() { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(UseSingleVector) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -41,7 +43,7 @@ public void ItThrowsExceptionWithInvalidIndexKind() public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string expectedIndexKind) { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(UseSingleVector) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -65,7 +67,7 @@ public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string ex public void ItThrowsExceptionWithInvalidDistanceFunction() { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(UseSingleVector) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -90,7 +92,7 @@ public void ItThrowsExceptionWithInvalidDistanceFunction() public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunction, string expectedDistanceFunction) { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(UseSingleVector) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -163,7 +165,7 @@ public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunct public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyType) { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(UseSingleVector) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs index 27cf9dc4d82c..446b0b56ed2d 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs @@ -31,7 +31,7 @@ public sealed class WeaviateVectorStoreRecordCollectionQueryBuilderTests } }; - private readonly VectorStoreRecordModel _model = new WeaviateModelBuilder() + private readonly VectorStoreRecordModel _model = new WeaviateModelBuilder(useSingleVector: false) .Build( typeof(VectorStoreGenericDataModel), new() diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs index f624599f5478..c313ea76c141 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs @@ -28,27 +28,10 @@ public sealed class WeaviateVectorStoreRecordMapperTests } }; - private readonly WeaviateVectorStoreRecordMapper _sut = - new( - "CollectionName", - new WeaviateModelBuilder() - .Build( - typeof(VectorStoreGenericDataModel), - new VectorStoreRecordDefinition - { - Properties = - [ - new VectorStoreRecordKeyProperty("HotelId", typeof(Guid)), - new VectorStoreRecordDataProperty("HotelName", typeof(string)), - new VectorStoreRecordDataProperty("Tags", typeof(List)), - new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory)) - ] - }, - s_jsonSerializerOptions), - s_jsonSerializerOptions); - - [Fact] - public void MapFromDataToStorageModelReturnsValidObject() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromDataToStorageModelReturnsValidObject(bool useSingleVector) { // Arrange var hotel = new WeaviateHotel @@ -59,8 +42,10 @@ public void MapFromDataToStorageModelReturnsValidObject() DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f]) }; + var sut = GetMapper(useSingleVector); + // Act - var document = this._sut.MapFromDataToStorageModel(hotel); + var document = sut.MapFromDataToStorageModel(hotel); // Assert Assert.NotNull(document); @@ -68,11 +53,16 @@ public void MapFromDataToStorageModelReturnsValidObject() Assert.Equal("55555555-5555-5555-5555-555555555555", document["id"]!.GetValue()); Assert.Equal("Test Name", document["properties"]!["hotelName"]!.GetValue()); Assert.Equal(["tag1", "tag2"], document["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); + + var vectorNode = useSingleVector ? document["vector"] : document["vectors"]!["descriptionEmbedding"]; + Assert.Equal([1f, 2f, 3f], document["vectors"]!["descriptionEmbedding"]!.AsArray().Select(l => l!.GetValue())); } - [Fact] - public void MapFromStorageToDataModelReturnsValidObject() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromStorageToDataModelReturnsValidObject(bool useSingleVector) { // Arrange var document = new JsonObject @@ -84,10 +74,22 @@ public void MapFromStorageToDataModelReturnsValidObject() document["properties"]!["hotelName"] = "Test Name"; document["properties"]!["tags"] = new JsonArray(new List { "tag1", "tag2" }.Select(l => JsonValue.Create(l)).ToArray()); - document["vectors"]!["descriptionEmbedding"] = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); + + var vectorNode = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); + + if (useSingleVector) + { + document["vector"] = vectorNode; + } + else + { + document["vectors"]!["descriptionEmbedding"] = vectorNode; + } + + var sut = GetMapper(useSingleVector); // Act - var hotel = this._sut.MapFromStorageToDataModel(document, new() { IncludeVectors = true }); + var hotel = sut.MapFromStorageToDataModel(document, new() { IncludeVectors = true }); // Assert Assert.NotNull(hotel); @@ -97,4 +99,27 @@ public void MapFromStorageToDataModelReturnsValidObject() Assert.Equal(["tag1", "tag2"], hotel.Tags); Assert.True(new ReadOnlyMemory([1f, 2f, 3f]).Span.SequenceEqual(hotel.DescriptionEmbedding!.Value.Span)); } + + #region private + + private static WeaviateVectorStoreRecordMapper GetMapper(bool useSingleVector) => new( + "CollectionName", + useSingleVector, + new WeaviateModelBuilder(useSingleVector) + .Build( + typeof(VectorStoreGenericDataModel), + new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("HotelId", typeof(Guid)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)), + new VectorStoreRecordDataProperty("Tags", typeof(List)), + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory)) + ] + }, + s_jsonSerializerOptions), + s_jsonSerializerOptions); + + #endregion } From cfe5f47df94759f184ecea657386304b16b2eaa6 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 14:13:43 -0700 Subject: [PATCH 02/15] Updated unit tests --- .../WeaviateModelBuilder.cs | 11 +++++++++++ ...WeaviateVectorStoreCollectionCreateMappingTests.cs | 2 -- .../WeaviateVectorStoreRecordMapperTests.cs | 2 +- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs index 967c9cd5738c..d7556713d07c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs @@ -30,16 +30,27 @@ private static VectorStoreRecordModelBuildingOptions GetModelBuildingOptions(boo [ typeof(string), typeof(bool), + typeof(bool?), typeof(int), + typeof(int?), typeof(long), + typeof(long?), typeof(short), + typeof(short?), typeof(byte), + typeof(byte?), typeof(float), + typeof(float?), typeof(double), + typeof(double?), typeof(decimal), + typeof(decimal?), typeof(DateTime), + typeof(DateTime?), typeof(DateTimeOffset), + typeof(DateTimeOffset?), typeof(Guid), + typeof(Guid?) ]; internal static readonly HashSet s_supportedVectorTypes = diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs index 6b3be9496b97..2572be1192ce 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -160,8 +160,6 @@ public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunct [InlineData(typeof(bool?), "boolean")] [InlineData(typeof(List), "boolean[]")] [InlineData(typeof(List), "boolean[]")] - [InlineData(typeof(object), "object")] - [InlineData(typeof(List), "object[]")] public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyType) { // Arrange diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs index c313ea76c141..817e8096cf9d 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs @@ -56,7 +56,7 @@ public void MapFromDataToStorageModelReturnsValidObject(bool useSingleVector) var vectorNode = useSingleVector ? document["vector"] : document["vectors"]!["descriptionEmbedding"]; - Assert.Equal([1f, 2f, 3f], document["vectors"]!["descriptionEmbedding"]!.AsArray().Select(l => l!.GetValue())); + Assert.Equal([1f, 2f, 3f], vectorNode!.AsArray().Select(l => l!.GetValue())); } [Theory] From ac4ccabbfdf1003295d116d634a44a8150ce80f1 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 14:27:38 -0700 Subject: [PATCH 03/15] Inverted boolean property --- .../WeaviateModelBuilder.cs | 6 +- .../WeaviateVectorStoreRecordCollection.cs | 4 +- ...viateVectorStoreRecordCollectionOptions.cs | 7 +-- .../WeaviateVectorStoreRecordMapper.cs | 56 +++++++++---------- .../WeaviateGenericDataModelMapperTests.cs | 8 +-- ...VectorStoreCollectionCreateMappingTests.cs | 12 ++-- ...rStoreRecordCollectionQueryBuilderTests.cs | 2 +- .../WeaviateVectorStoreRecordMapperTests.cs | 22 ++++---- 8 files changed, 58 insertions(+), 59 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs index d7556713d07c..680d9ad3d40c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs @@ -6,15 +6,15 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; -internal class WeaviateModelBuilder(bool useSingleVector) : VectorStoreRecordJsonModelBuilder(GetModelBuildingOptions(useSingleVector)) +internal class WeaviateModelBuilder(bool hasNamedVectors) : VectorStoreRecordJsonModelBuilder(GetModelBuildingOptions(hasNamedVectors)) { - private static VectorStoreRecordModelBuildingOptions GetModelBuildingOptions(bool useSingleVector) + private static VectorStoreRecordModelBuildingOptions GetModelBuildingOptions(bool hasNamedVectors) { return new() { RequiresAtLeastOneVector = false, SupportsMultipleKeys = false, - SupportsMultipleVectors = !useSingleVector, + SupportsMultipleVectors = hasNamedVectors, SupportedKeyPropertyTypes = [typeof(Guid)], SupportedDataPropertyTypes = s_supportedDataTypes, diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 9d871a5976f8..6a297c19becb 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -95,7 +95,7 @@ public WeaviateVectorStoreRecordCollection( this.CollectionName = collectionName; this._options = options ?? new(); this._apiKey = this._options.ApiKey; - this._model = new WeaviateModelBuilder(this._options.UseSingleVector) + this._model = new WeaviateModelBuilder(this._options.HasNamedVectors) .Build(typeof(TRecord), this._options.VectorStoreRecordDefinition, s_jsonSerializerOptions); // Assign mapper. @@ -470,7 +470,7 @@ private IVectorStoreRecordMapper InitializeMapper() return new WeaviateVectorStoreRecordMapper( this.CollectionName, - this._options.UseSingleVector, + this._options.HasNamedVectors, this._model, s_jsonSerializerOptions); } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs index 7ef5b570387a..7e125d36f70c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs @@ -41,10 +41,9 @@ public sealed class WeaviateVectorStoreRecordCollectionOptions public string? ApiKey { get; set; } = null; /// - /// Boolean flag which indicates whether a single vector data schema should be used in Weaviate collection. - /// By default it's , meaning that multiple vector data schema will be used (also known as named vectors). + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// Defaults to multiple named vectors. /// . - /// When single vector data schema is enabled, only one vector property will be used with name "vector". /// - public bool UseSingleVector { get; set; } = false; + public bool HasNamedVectors { get; set; } = true; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs index f148ea2eb9b1..677aad300a22 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs @@ -13,7 +13,7 @@ internal sealed class WeaviateVectorStoreRecordMapper : IVectorStoreRec #pragma warning restore CS0618 { private readonly string _collectionName; - private readonly bool _useSingleVector; + private readonly bool _hasNamedVectors; private readonly VectorStoreRecordModel _model; private readonly JsonSerializerOptions _jsonSerializerOptions; @@ -21,18 +21,18 @@ internal sealed class WeaviateVectorStoreRecordMapper : IVectorStoreRec public WeaviateVectorStoreRecordMapper( string collectionName, - bool useSingleVector, + bool hasNamedVectors, VectorStoreRecordModel model, JsonSerializerOptions jsonSerializerOptions) { this._collectionName = collectionName; - this._useSingleVector = useSingleVector; + this._hasNamedVectors = hasNamedVectors; this._model = model; this._jsonSerializerOptions = jsonSerializerOptions; - this._vectorPropertyName = useSingleVector ? - WeaviateConstants.ReservedSingleVectorPropertyName : - WeaviateConstants.ReservedVectorPropertyName; + this._vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; } public JsonObject MapFromDataToStorageModel(TRecord dataModel) @@ -65,17 +65,7 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel) } // Populate vector properties. - if (this._useSingleVector) - { - var vectorProperty = this._model.VectorProperty; - var node = jsonNodeDataModel[vectorProperty.StorageName]; - - if (node is not null) - { - weaviateObjectModel[this._vectorPropertyName] = node.DeepClone(); - } - } - else + if (this._hasNamedVectors) { foreach (var property in this._model.VectorProperties) { @@ -87,6 +77,16 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel) } } } + else + { + var vectorProperty = this._model.VectorProperty; + var node = jsonNodeDataModel[vectorProperty.StorageName]; + + if (node is not null) + { + weaviateObjectModel[this._vectorPropertyName] = node.DeepClone(); + } + } return weaviateObjectModel; } @@ -119,17 +119,7 @@ public TRecord MapFromStorageToDataModel(JsonObject storageModel, StorageToDataM // Populate vector properties. if (options.IncludeVectors) { - if (this._useSingleVector) - { - var vectorProperty = this._model.VectorProperty; - var node = storageModel[this._vectorPropertyName]; - - if (node is not null) - { - jsonNodeDataModel[vectorProperty.StorageName] = node.DeepClone(); - } - } - else + if (this._hasNamedVectors) { foreach (var property in this._model.VectorProperties) { @@ -141,6 +131,16 @@ public TRecord MapFromStorageToDataModel(JsonObject storageModel, StorageToDataM } } } + else + { + var vectorProperty = this._model.VectorProperty; + var node = storageModel[this._vectorPropertyName]; + + if (node is not null) + { + jsonNodeDataModel[vectorProperty.StorageName] = node.DeepClone(); + } + } } return jsonNodeDataModel.Deserialize(this._jsonSerializerOptions)!; diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs index 50b70b38ad4d..0a5d2e057dd7 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateGenericDataModelMapperTests.cs @@ -18,7 +18,7 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateGenericDataModelMapperTests { - private const bool UseSingleVector = false; + private const bool HasNamedVectors = true; private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() { @@ -31,7 +31,7 @@ public sealed class WeaviateGenericDataModelMapperTests } }; - private static readonly VectorStoreRecordModel s_model = new WeaviateModelBuilder(UseSingleVector) + private static readonly VectorStoreRecordModel s_model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -353,7 +353,7 @@ public void MapFromDataToStorageModelSkipsMissingProperties() ] }; - var model = new WeaviateModelBuilder(UseSingleVector).Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); + var model = new WeaviateModelBuilder(HasNamedVectors).Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); var key = new Guid("55555555-5555-5555-5555-555555555555"); @@ -384,7 +384,7 @@ public void MapFromStorageToDataModelSkipsMissingProperties() ] }; - var model = new WeaviateModelBuilder(UseSingleVector).Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); + var model = new WeaviateModelBuilder(HasNamedVectors).Build(typeof(VectorStoreGenericDataModel), recordDefinition, s_jsonSerializerOptions); var key = new Guid("55555555-5555-5555-5555-555555555555"); diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs index 2572be1192ce..f794c30736b8 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -14,13 +14,13 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateVectorStoreCollectionCreateMappingTests { - private const bool UseSingleVector = false; + private const bool HasNamedVectors = true; [Fact] public void ItThrowsExceptionWithInvalidIndexKind() { // Arrange - var model = new WeaviateModelBuilder(UseSingleVector) + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -43,7 +43,7 @@ public void ItThrowsExceptionWithInvalidIndexKind() public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string expectedIndexKind) { // Arrange - var model = new WeaviateModelBuilder(UseSingleVector) + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -67,7 +67,7 @@ public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string ex public void ItThrowsExceptionWithInvalidDistanceFunction() { // Arrange - var model = new WeaviateModelBuilder(UseSingleVector) + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -92,7 +92,7 @@ public void ItThrowsExceptionWithInvalidDistanceFunction() public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunction, string expectedDistanceFunction) { // Arrange - var model = new WeaviateModelBuilder(UseSingleVector) + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition @@ -163,7 +163,7 @@ public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunct public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyType) { // Arrange - var model = new WeaviateModelBuilder(UseSingleVector) + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs index 446b0b56ed2d..cbdf12da4b2d 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs @@ -31,7 +31,7 @@ public sealed class WeaviateVectorStoreRecordCollectionQueryBuilderTests } }; - private readonly VectorStoreRecordModel _model = new WeaviateModelBuilder(useSingleVector: false) + private readonly VectorStoreRecordModel _model = new WeaviateModelBuilder(hasNamedVectors: true) .Build( typeof(VectorStoreGenericDataModel), new() diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs index 817e8096cf9d..5cabb80396ed 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs @@ -31,7 +31,7 @@ public sealed class WeaviateVectorStoreRecordMapperTests [Theory] [InlineData(true)] [InlineData(false)] - public void MapFromDataToStorageModelReturnsValidObject(bool useSingleVector) + public void MapFromDataToStorageModelReturnsValidObject(bool hasNamedVectors) { // Arrange var hotel = new WeaviateHotel @@ -42,7 +42,7 @@ public void MapFromDataToStorageModelReturnsValidObject(bool useSingleVector) DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f]) }; - var sut = GetMapper(useSingleVector); + var sut = GetMapper(hasNamedVectors); // Act var document = sut.MapFromDataToStorageModel(hotel); @@ -54,7 +54,7 @@ public void MapFromDataToStorageModelReturnsValidObject(bool useSingleVector) Assert.Equal("Test Name", document["properties"]!["hotelName"]!.GetValue()); Assert.Equal(["tag1", "tag2"], document["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); - var vectorNode = useSingleVector ? document["vector"] : document["vectors"]!["descriptionEmbedding"]; + var vectorNode = hasNamedVectors ? document["vectors"]!["descriptionEmbedding"] : document["vector"]; Assert.Equal([1f, 2f, 3f], vectorNode!.AsArray().Select(l => l!.GetValue())); } @@ -62,7 +62,7 @@ public void MapFromDataToStorageModelReturnsValidObject(bool useSingleVector) [Theory] [InlineData(true)] [InlineData(false)] - public void MapFromStorageToDataModelReturnsValidObject(bool useSingleVector) + public void MapFromStorageToDataModelReturnsValidObject(bool hasNamedVectors) { // Arrange var document = new JsonObject @@ -77,16 +77,16 @@ public void MapFromStorageToDataModelReturnsValidObject(bool useSingleVector) var vectorNode = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); - if (useSingleVector) + if (hasNamedVectors) { - document["vector"] = vectorNode; + document["vectors"]!["descriptionEmbedding"] = vectorNode; } else { - document["vectors"]!["descriptionEmbedding"] = vectorNode; + document["vector"] = vectorNode; } - var sut = GetMapper(useSingleVector); + var sut = GetMapper(hasNamedVectors); // Act var hotel = sut.MapFromStorageToDataModel(document, new() { IncludeVectors = true }); @@ -102,10 +102,10 @@ public void MapFromStorageToDataModelReturnsValidObject(bool useSingleVector) #region private - private static WeaviateVectorStoreRecordMapper GetMapper(bool useSingleVector) => new( + private static WeaviateVectorStoreRecordMapper GetMapper(bool hasNamedVectors) => new( "CollectionName", - useSingleVector, - new WeaviateModelBuilder(useSingleVector) + hasNamedVectors, + new WeaviateModelBuilder(hasNamedVectors) .Build( typeof(VectorStoreGenericDataModel), new VectorStoreRecordDefinition From 71780b66be7b77c315c70ad324af44195da38ea3 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:04:36 -0700 Subject: [PATCH 04/15] Updates to integration tests --- .../HttpV2/WeaviateGetCollectionsRequest.cs | 2 +- .../Collections/CollectionConformanceTests.cs | 36 ++----------------- .../CRUD/WeaviateNoVectorConformanceTests.cs | 2 +- ...ollectionConformanceTests_NamedVectors.cs} | 6 ++-- ...ollectionConformanceTests_UnnamedVector.cs | 12 +++++++ .../Filter/WeaviateBasicFilterTests.cs | 2 +- ...ctorizedHybridSearchTests_NamedVectors.cs} | 14 ++++---- ...torizedHybridSearchTests_UnnamedVectors.cs | 34 ++++++++++++++++++ ...ture.cs => WeaviateNamedVectorsFixture.cs} | 4 +-- .../Support/WeaviateTestStore.cs | 8 ++--- .../Support/WeaviateUnnamedVectorFixture.cs | 14 ++++++++ 11 files changed, 82 insertions(+), 52 deletions(-) rename dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/{WeaviateCollectionConformanceTests.cs => WeaviateCollectionConformanceTests_NamedVectors.cs} (68%) create mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs rename dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/{WeaviateKeywordVectorizedHybridSearchTests.cs => WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs} (76%) create mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs rename dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/{WeaviateFixture.cs => WeaviateNamedVectorsFixture.cs} (76%) create mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs index f31017ca8685..40012278a076 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs @@ -10,6 +10,6 @@ internal sealed class WeaviateGetCollectionsRequest public HttpRequestMessage Build() { - return HttpRequest.CreateGetRequest(ApiRoute, this); + return HttpRequest.CreateGetRequest(ApiRoute); } } diff --git a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs index 3af9eae17e58..a3d67bb4410b 100644 --- a/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/VectorDataIntegrationTests/Collections/CollectionConformanceTests.cs @@ -86,17 +86,7 @@ private async Task CreateCollection() try { Assert.True(await collection.CollectionExistsAsync()); - -#pragma warning disable MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; -#pragma warning restore MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - - Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreSystemName); - Assert.NotNull(collectionMetadata.VectorStoreName); - Assert.NotNull(collectionMetadata.CollectionName); - - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); } finally { @@ -113,17 +103,7 @@ private async Task CreateCollectionIfNotExistsMoreThanOnce() try { Assert.True(await collection.CollectionExistsAsync()); - -#pragma warning disable MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; -#pragma warning restore MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - - Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreSystemName); - Assert.NotNull(collectionMetadata.VectorStoreName); - Assert.NotNull(collectionMetadata.CollectionName); - - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); await collection.CreateCollectionIfNotExistsAsync(); } @@ -142,17 +122,7 @@ private async Task CreateCollectionMoreThanOnce() try { Assert.True(await collection.CollectionExistsAsync()); - -#pragma warning disable MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - var collectionMetadata = collection.GetService(typeof(VectorStoreRecordCollectionMetadata)) as VectorStoreRecordCollectionMetadata; -#pragma warning restore MEVD9000 // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. - - Assert.NotNull(collectionMetadata); - Assert.NotNull(collectionMetadata.VectorStoreSystemName); - Assert.NotNull(collectionMetadata.VectorStoreName); - Assert.NotNull(collectionMetadata.CollectionName); - - Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collectionMetadata.CollectionName)); + Assert.True(await fixture.TestStore.DefaultVectorStore.ListCollectionNamesAsync().ContainsAsync(collection.CollectionName)); await collection.CreateCollectionIfNotExistsAsync(); diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs index 016ab64870e6..20d5888ad851 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs @@ -12,7 +12,7 @@ public class WeaviateNoVectorConformanceTests(WeaviateNoVectorConformanceTests.F { public new class Fixture : NoVectorConformanceTests.Fixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; /// /// Weaviate collections must start with an uppercase letter. diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs similarity index 68% rename from dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs rename to dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs index e839b02ad942..e6480c4edbab 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs @@ -1,12 +1,12 @@ // Copyright (c) Microsoft. All rights reserved. -using VectorDataSpecificationTests.Collections; using WeaviateIntegrationTests.Support; +using VectorDataSpecificationTests.Collections; using Xunit; namespace WeaviateIntegrationTests.Collections; -public class WeaviateCollectionConformanceTests(WeaviateFixture fixture) - : CollectionConformanceTests(fixture), IClassFixture +public class WeaviateCollectionConformanceTests_NamedVectors(WeaviateNamedVectorsFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs new file mode 100644 index 000000000000..c4fafaa94f1e --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. + +using WeaviateIntegrationTests.Support; +using VectorDataSpecificationTests.Collections; +using Xunit; + +namespace WeaviateIntegrationTests.Collections; + +public class WeaviateCollectionConformanceTests_UnnamedVector(WeaviateUnnamedVectorFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs index 6238ca6d9b6a..f5f34a693587 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs @@ -65,7 +65,7 @@ public override Task Equal_with_string_is_not_Contains() public new class Fixture : BasicFilterTests.Fixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs similarity index 76% rename from dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs rename to dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs index 30d6bc0516f5..b8ca5dfa0fe0 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs @@ -7,16 +7,16 @@ namespace WeaviateIntegrationTests.HybridSearch; -public class WeaviateKeywordVectorizedHybridSearchTests( - WeaviateKeywordVectorizedHybridSearchTests.VectorAndStringFixture vectorAndStringFixture, - WeaviateKeywordVectorizedHybridSearchTests.MultiTextFixture multiTextFixture) +public class WeaviateKeywordVectorizedHybridSearchTests_NamedVectors( + WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.VectorAndStringFixture vectorAndStringFixture, + WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.MultiTextFixture multiTextFixture) : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), - IClassFixture, - IClassFixture + IClassFixture, + IClassFixture { public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; @@ -25,7 +25,7 @@ public class WeaviateKeywordVectorizedHybridSearchTests( public new class MultiTextFixture : KeywordVectorizedHybridSearchComplianceTests.MultiTextFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs new file mode 100644 index 000000000000..e06472c8b59a --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.HybridSearch; +using VectorDataSpecificationTests.Support; +using WeaviateIntegrationTests.Support; +using Xunit; + +namespace WeaviateIntegrationTests.HybridSearch; + +public class WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors( + WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.VectorAndStringFixture vectorAndStringFixture, + WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.MultiTextFixture multiTextFixture) + : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + protected override string CollectionName => "VectorAndStringHybridSearch"; + } + + public new class MultiTextFixture : KeywordVectorizedHybridSearchComplianceTests.MultiTextFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + protected override string CollectionName => "MultiTextHybridSearch"; + } +} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs similarity index 76% rename from dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateFixture.cs rename to dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs index ac3b64f89006..a132cbed0ccb 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs @@ -4,9 +4,9 @@ namespace WeaviateIntegrationTests.Support; -public class WeaviateFixture : VectorStoreFixture +public class WeaviateNamedVectorsFixture : VectorStoreFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs index d112a2abfe49..70a089c03f5b 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs @@ -12,9 +12,11 @@ namespace WeaviateIntegrationTests.Support; public sealed class WeaviateTestStore : TestStore { - public static WeaviateTestStore Instance { get; } = new(); + public static WeaviateTestStore NamedVectorsInstance { get; } = new(hasNamedVectors: true); + public static WeaviateTestStore UnnamedVectorInstance { get; } = new(hasNamedVectors: false); private readonly WeaviateContainer _container = new WeaviateBuilder().Build(); + private readonly bool _hasNamedVectors; public HttpClient? _httpClient { get; private set; } private WeaviateVectorStore? _defaultVectorStore; @@ -25,9 +27,7 @@ public sealed class WeaviateTestStore : TestStore public WeaviateVectorStore GetVectorStore(WeaviateVectorStoreOptions options) => new(this.Client, options); - private WeaviateTestStore() - { - } + private WeaviateTestStore(bool hasNamedVectors) => this._hasNamedVectors = hasNamedVectors; protected override async Task StartAsync() { diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs new file mode 100644 index 000000000000..01cd29b92417 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft. All rights reserved. + +using VectorDataSpecificationTests.Support; + +namespace WeaviateIntegrationTests.Support; + +public class WeaviateUnnamedVectorFixture : VectorStoreFixture +{ + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + + // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. + // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names + public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; +} From 2c31174569187b86eeb41f1804d570f292fdaff2 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:31:58 -0700 Subject: [PATCH 05/15] Fixed HTTP communication logic --- .../WeaviateVectorStore.cs | 3 ++ .../WeaviateVectorStoreRecordCollection.cs | 31 ++++++++++++++----- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs index 4d9f0236beab..a3c6c8051b0d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -91,6 +91,9 @@ public async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancel try { var httpResponse = await this._httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); + + httpResponse.EnsureSuccessStatusCode(); + var httpResponseContent = await httpResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); collectionsResponse = JsonSerializer.Deserialize(httpResponseContent)!; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 6a297c19becb..c77383a327b4 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -136,7 +136,7 @@ public Task CreateCollectionAsync(CancellationToken cancellationToken = default) var request = new WeaviateCreateCollectionSchemaRequest(schema).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -158,7 +158,7 @@ public Task DeleteCollectionAsync(CancellationToken cancellationToken = default) { var request = new WeaviateDeleteCollectionSchemaRequest(this.CollectionName).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -171,7 +171,7 @@ public Task DeleteAsync(Guid key, CancellationToken cancellationToken = default) { var request = new WeaviateDeleteObjectRequest(this.CollectionName, key).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -196,7 +196,7 @@ public Task DeleteAsync(IEnumerable keys, CancellationToken cancellationTo var request = new WeaviateDeleteObjectBatchRequest(match).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -388,7 +388,10 @@ private async Task> ExecuteQueryAsync(string query, return new VectorSearchResults(mappedResults.ToAsyncEnumerable()); } - private Task ExecuteRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + private async Task ExecuteRequestAsync( + HttpRequestMessage request, + bool ensureSuccessStatusCode = true, + CancellationToken cancellationToken = default) { request.RequestUri = new Uri(this._endpoint, request.RequestUri!); @@ -397,12 +400,21 @@ private Task ExecuteRequestAsync(HttpRequestMessage request request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", this._apiKey); } - return this._httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken); + var response = await this._httpClient + .SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken) + .ConfigureAwait(false); + + if (ensureSuccessStatusCode) + { + response.EnsureSuccessStatusCode(); + } + + return response; } private async Task<(TResponse?, string)> ExecuteRequestWithResponseContentAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var response = await this.ExecuteRequestAsync(request, cancellationToken).ConfigureAwait(false); + var response = await this.ExecuteRequestAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); @@ -420,12 +432,15 @@ private Task ExecuteRequestAsync(HttpRequestMessage request private async Task ExecuteRequestWithNotFoundHandlingAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var response = await this.ExecuteRequestAsync(request, cancellationToken).ConfigureAwait(false); + var response = await this.ExecuteRequestAsync(request, ensureSuccessStatusCode: false, cancellationToken: cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) { return default; } + response.EnsureSuccessStatusCode(); + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var responseModel = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); From d20aa196ab467e3a03a407639d503052e82bdabe Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:36:52 -0700 Subject: [PATCH 06/15] Small naming fix --- ...eywordVectorizedHybridSearchTests_UnnamedVector.cs} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/{WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs => WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs} (82%) diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs similarity index 82% rename from dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs rename to dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs index e06472c8b59a..a4c6db57a4c9 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs @@ -7,12 +7,12 @@ namespace WeaviateIntegrationTests.HybridSearch; -public class WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors( - WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.VectorAndStringFixture vectorAndStringFixture, - WeaviateKeywordVectorizedHybridSearchTests_UnnamedVectors.MultiTextFixture multiTextFixture) +public class WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector( + WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.VectorAndStringFixture vectorAndStringFixture, + WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.MultiTextFixture multiTextFixture) : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), - IClassFixture, - IClassFixture + IClassFixture, + IClassFixture { public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture { From 6e86703f7db12e10dc3dacb5eaddb1777e44c1f8 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:40:29 -0700 Subject: [PATCH 07/15] Fixed formatting --- .../WeaviateCollectionConformanceTests_UnnamedVector.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs index c4fafaa94f1e..a070c49ed28c 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using WeaviateIntegrationTests.Support; using VectorDataSpecificationTests.Collections; +using WeaviateIntegrationTests.Support; using Xunit; namespace WeaviateIntegrationTests.Collections; From 9e19ae04407941eef5947d4d0e36c953c9a7235c Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:40:39 -0700 Subject: [PATCH 08/15] Fixed formatting --- .../WeaviateCollectionConformanceTests_NamedVectors.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs index e6480c4edbab..b78c7f705b59 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using WeaviateIntegrationTests.Support; using VectorDataSpecificationTests.Collections; +using WeaviateIntegrationTests.Support; using Xunit; namespace WeaviateIntegrationTests.Collections; From e90e1a1c847c4d6361b910d5608ea42071238951 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Tue, 8 Apr 2025 15:55:55 -0700 Subject: [PATCH 09/15] More fixes --- .../Connectors.Memory.Weaviate/WeaviateVectorStore.cs | 3 ++- .../WeaviateVectorStoreOptions.cs | 7 +++++++ .../WeaviateIntegrationTests/Support/WeaviateTestStore.cs | 5 +---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs index a3c6c8051b0d..884758b39cdf 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -76,7 +76,8 @@ public IVectorStoreRecordCollection GetCollection( { VectorStoreRecordDefinition = vectorStoreRecordDefinition, Endpoint = this._options.Endpoint, - ApiKey = this._options.ApiKey + ApiKey = this._options.ApiKey, + HasNamedVectors = this._options.HasNamedVectors }) as IVectorStoreRecordCollection; return recordCollection!; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs index ae73e7989d82..a3b87481bd1e 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs @@ -27,4 +27,11 @@ public sealed class WeaviateVectorStoreOptions /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. /// public string? ApiKey { get; set; } = null; + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// Defaults to multiple named vectors. + /// . + /// + public bool HasNamedVectors { get; set; } = true; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs index 70a089c03f5b..76aa72077a3a 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs @@ -24,16 +24,13 @@ public sealed class WeaviateTestStore : TestStore public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized"); - public WeaviateVectorStore GetVectorStore(WeaviateVectorStoreOptions options) - => new(this.Client, options); - private WeaviateTestStore(bool hasNamedVectors) => this._hasNamedVectors = hasNamedVectors; protected override async Task StartAsync() { await this._container.StartAsync(); this._httpClient = new HttpClient { BaseAddress = new Uri($"http://localhost:{this._container.GetMappedPublicPort(WeaviateBuilder.WeaviateHttpPort)}/v1/") }; - this._defaultVectorStore = new(this._httpClient); + this._defaultVectorStore = new(this._httpClient, new() { HasNamedVectors = this._hasNamedVectors }); } protected override Task StopAsync() From 1c14eeb048e2618db5ac85aa52b37ff8cb0974fd Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 10 Apr 2025 08:44:08 -0700 Subject: [PATCH 10/15] Small fixes after merge --- .../WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs | 2 +- .../Support/WeaviateDynamicDataModelFixture.cs | 2 +- .../Support/WeaviateSimpleModelFixture.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs index 312ac21b8372..c03a8fd8076b 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs @@ -65,7 +65,7 @@ public override Task Equal_with_string_is_not_Contains() public new class Fixture : BasicQueryTests.QueryFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs index cd69271c5a12..7e8ec36a04fa 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs @@ -6,7 +6,7 @@ namespace WeaviateIntegrationTests.Support; public class WeaviateDynamicDataModelFixture : DynamicDataModelFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs index 0fe7c713e46b..257eecec88ce 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs @@ -6,7 +6,7 @@ namespace WeaviateIntegrationTests.Support; public class WeaviateSimpleModelFixture : SimpleModelFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names From 133ad868f3d8c762d06976002ed2d4f45186f46d Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:03:37 -0700 Subject: [PATCH 11/15] More updates to unnamed vector logic and tests --- .../ModelV2/WeaviateCollectionSchema.cs | 9 ++ .../WeaviateCollectionSchemaVectorConfig.cs | 4 +- .../WeaviateConstants.cs | 3 + .../WeaviateDynamicDataModelMapper.cs | 59 ++++++++++-- ...viateVectorStoreCollectionCreateMapping.cs | 31 +++++-- ...viateVectorStoreCollectionSearchMapping.cs | 13 ++- .../WeaviateVectorStoreRecordCollection.cs | 24 +++-- ...VectorStoreRecordCollectionQueryBuilder.cs | 42 ++++++--- .../WeaviateDynamicDataModelMapperTests.cs | 91 +++++++++++++++++-- ...VectorStoreCollectionCreateMappingTests.cs | 58 ++++++++++-- ...VectorStoreCollectionSearchMappingTests.cs | 33 +++++-- ...rStoreRecordCollectionQueryBuilderTests.cs | 33 ++++--- .../CRUD/WeaviateBatchConformanceTests.cs | 9 +- .../WeaviateDynamicRecordConformanceTests.cs | 9 +- .../CRUD/WeaviateRecordConformanceTests.cs | 9 +- ... => WeaviateCollectionConformanceTests.cs} | 9 +- ...ollectionConformanceTests_UnnamedVector.cs | 12 --- .../Filter/WeaviateBasicFilterTests.cs | 2 + ...iateKeywordVectorizedHybridSearchTests.cs} | 26 ++++++ ...ectorizedHybridSearchTests_NamedVectors.cs | 34 ------- .../Support/TestContainer/WeaviateBuilder.cs | 2 +- .../WeaviateDynamicDataModelFixture.cs | 14 ++- .../Support/WeaviateNamedVectorsFixture.cs | 14 --- .../Support/WeaviateSimpleModelFixture.cs | 16 +++- .../Support/WeaviateUnnamedVectorFixture.cs | 14 --- ...orSearchDistanceFunctionComplianceTests.cs | 42 +++++++++ 26 files changed, 447 insertions(+), 165 deletions(-) rename dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/{WeaviateCollectionConformanceTests_NamedVectors.cs => WeaviateCollectionConformanceTests.cs} (52%) delete mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs rename dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/{WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs => WeaviateKeywordVectorizedHybridSearchTests.cs} (54%) delete mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs delete mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs delete mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs create mode 100644 dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs index e0f403ddb0e8..c6122eea8967 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs @@ -21,4 +21,13 @@ public WeaviateCollectionSchema(string collectionName) [JsonPropertyName("properties")] public List Properties { get; set; } = []; + + [JsonPropertyName("vectorizer")] + public string Vectorizer { get; set; } = WeaviateConstants.DefaultVectorizer; + + [JsonPropertyName("vectorIndexType")] + public string? VectorIndexType { get; set; } + + [JsonPropertyName("vectorIndexConfig")] + public WeaviateCollectionSchemaVectorIndexConfig? VectorIndexConfig { get; set; } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs index 75bd33471eb7..77830facd893 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs @@ -7,10 +7,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; internal sealed class WeaviateCollectionSchemaVectorConfig { - private const string DefaultVectorizer = "none"; - [JsonPropertyName("vectorizer")] - public Dictionary Vectorizer { get; set; } = new() { [DefaultVectorizer] = null }; + public Dictionary Vectorizer { get; set; } = new() { [WeaviateConstants.DefaultVectorizer] = null }; [JsonPropertyName("vectorIndexType")] public string? VectorIndexType { get; set; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs index bcb4d1f53b56..f98d4a6304cd 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs @@ -30,4 +30,7 @@ internal sealed class WeaviateConstants /// Additional properties property name in Weaviate. internal const string AdditionalPropertiesPropertyName = "_additional"; + + /// Default vectorizer for vector properties in Weaviate. + internal const string DefaultVectorizer = "none"; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs index 233d2e0f31c7..506956eb62f3 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs @@ -25,20 +25,33 @@ internal sealed class WeaviateDynamicDataModelMapper : IVectorStoreRecordMapper< /// A for serialization/deserialization of record properties. private readonly JsonSerializerOptions _jsonSerializerOptions; + /// Gets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + private readonly bool _hasNamedVectors; + + /// Gets a vector property named used in Weaviate collection. + private readonly string _vectorPropertyName; + /// /// Initializes a new instance of the class. /// /// The name of the Weaviate collection + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection /// The model /// A for serialization/deserialization of record properties. public WeaviateDynamicDataModelMapper( string collectionName, + bool hasNamedVectors, VectorStoreRecordModel model, JsonSerializerOptions jsonSerializerOptions) { this._collectionName = collectionName; + this._hasNamedVectors = hasNamedVectors; this._model = model; this._jsonSerializerOptions = jsonSerializerOptions; + + this._vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; } public JsonObject MapFromDataToStorageModel(Dictionary dataModel) @@ -61,14 +74,28 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode } // Populate vector properties. - var vectorObject = new JsonObject(); - foreach (var property in this._model.VectorProperties) + JsonNode? vectorObject = null; + + if (this._hasNamedVectors) { - if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + vectorObject = new JsonObject(); + foreach (var property in this._model.VectorProperties) { - vectorObject[property.StorageName] = vectorValue is null + if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + { + vectorObject[property.StorageName] = vectorValue is null + ? null + : JsonSerializer.SerializeToNode(vectorValue, property.Type, this._jsonSerializerOptions); + } + } + } + else + { + if (dataModel.TryGetValue(this._model.VectorProperty.ModelName, out var vectorValue)) + { + vectorObject = vectorValue is null ? null - : JsonSerializer.SerializeToNode(vectorValue, property.Type, this._jsonSerializerOptions); + : JsonSerializer.SerializeToNode(vectorValue, this._model.VectorProperty.Type, this._jsonSerializerOptions); } } @@ -77,7 +104,7 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode { WeaviateConstants.CollectionPropertyName, JsonValue.Create(this._collectionName) }, { WeaviateConstants.ReservedKeyPropertyName, keyObject }, { WeaviateConstants.ReservedDataPropertyName, dataObject }, - { WeaviateConstants.ReservedVectorPropertyName, vectorObject }, + { this._vectorPropertyName, vectorObject }, }; } @@ -111,13 +138,25 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode // Populate vector properties. if (options.IncludeVectors) { - foreach (var property in this._model.VectorProperties) + if (this._hasNamedVectors) + { + foreach (var property in this._model.VectorProperties) + { + var jsonObject = storageModel[WeaviateConstants.ReservedVectorPropertyName] as JsonObject; + + if (jsonObject is not null && jsonObject.TryGetPropertyValue(property.StorageName, out var vectorValue)) + { + result.Add(property.ModelName, vectorValue.Deserialize(property.Type, this._jsonSerializerOptions)); + } + } + } + else { - var jsonObject = storageModel[WeaviateConstants.ReservedVectorPropertyName] as JsonObject; + var jsonObject = storageModel[WeaviateConstants.ReservedSingleVectorPropertyName]; - if (jsonObject is not null && jsonObject.TryGetPropertyValue(property.StorageName, out var vectorValue)) + if (jsonObject is not null) { - result.Add(property.ModelName, vectorValue.Deserialize(property.Type, this._jsonSerializerOptions)); + result.Add(this._model.VectorProperty.ModelName, jsonObject.Deserialize(this._model.VectorProperty.Type, this._jsonSerializerOptions)); } } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs index 2448ceea8f18..852339436432 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs @@ -19,9 +19,10 @@ internal static class WeaviateVectorStoreCollectionCreateMapping /// Maps record type properties to Weaviate collection schema for collection creation. /// /// The name of the vector store collection. + /// Gets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. /// The model. /// Weaviate collection schema. - public static WeaviateCollectionSchema MapToSchema(string collectionName, VectorStoreRecordModel model) + public static WeaviateCollectionSchema MapToSchema(string collectionName, bool hasNamedVectors, VectorStoreRecordModel model) { var schema = new WeaviateCollectionSchema(collectionName); @@ -38,16 +39,28 @@ public static WeaviateCollectionSchema MapToSchema(string collectionName, Vector } // Handle vector properties. - foreach (var property in model.VectorProperties) + if (hasNamedVectors) { - schema.VectorConfigurations.Add(property.StorageName, new WeaviateCollectionSchemaVectorConfig + foreach (var property in model.VectorProperties) { - VectorIndexType = MapIndexKind(property.IndexKind, property.StorageName), - VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + schema.VectorConfigurations.Add(property.StorageName, new WeaviateCollectionSchemaVectorConfig { - Distance = MapDistanceFunction(property.DistanceFunction, property.StorageName) - } - }); + VectorIndexType = MapIndexKind(property.IndexKind, property.StorageName), + VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(property.DistanceFunction, property.StorageName) + } + }); + } + } + else + { + var vectorProperty = model.VectorProperty; + schema.VectorIndexType = MapIndexKind(vectorProperty.IndexKind, vectorProperty.StorageName); + schema.VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(vectorProperty.DistanceFunction, vectorProperty.StorageName) + }; } return schema; @@ -110,7 +123,7 @@ private static string MapDistanceFunction(string? distanceFunction, string vecto DistanceFunction.EuclideanSquaredDistance => EuclideanSquared, DistanceFunction.Hamming => Hamming, DistanceFunction.ManhattanDistance => Manhattan, - _ => throw new InvalidOperationException( + _ => throw new NotSupportedException( $"Distance function '{distanceFunction}' on {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " + $"Supported distance functions: {string.Join(", ", DistanceFunction.CosineDistance, diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs index 3842a3aded97..02ec36be81a1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs @@ -13,7 +13,10 @@ internal static class WeaviateVectorStoreCollectionSearchMapping /// /// Maps vector search result to the format, which is processable by . /// - public static (JsonObject StorageModel, double? Score) MapSearchResult(JsonNode result, string scorePropertyName) + public static (JsonObject StorageModel, double? Score) MapSearchResult( + JsonNode result, + string scorePropertyName, + bool hasNamedVectors) { var additionalProperties = result[WeaviateConstants.AdditionalPropertiesPropertyName]; @@ -25,14 +28,18 @@ public static (JsonObject StorageModel, double? Score) MapSearchResult(JsonNode _ => null }; + var vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; + var id = additionalProperties?[WeaviateConstants.ReservedKeyPropertyName]; - var vectors = additionalProperties?[WeaviateConstants.ReservedVectorPropertyName]; + var vectors = additionalProperties?[vectorPropertyName]; var storageModel = new JsonObject { { WeaviateConstants.ReservedKeyPropertyName, id?.DeepClone() }, { WeaviateConstants.ReservedDataPropertyName, result?.DeepClone() }, - { WeaviateConstants.ReservedVectorPropertyName, vectors?.DeepClone() }, + { vectorPropertyName, vectors?.DeepClone() }, }; return (storageModel, score); diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 0adf9872d421..c083d511d168 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -140,10 +140,13 @@ public Task CreateCollectionAsync(CancellationToken cancellationToken = default) { const string OperationName = "CreateCollectionSchema"; + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + this.CollectionName, + this._options.HasNamedVectors, + this._model); + return this.RunOperationAsync(OperationName, () => { - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(this.CollectionName, this._model); - var request = new WeaviateCreateCollectionSchemaRequest(schema).Build(); return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); @@ -346,7 +349,8 @@ public async Task> VectorizedSearchAsync( s_jsonSerializerOptions, top, searchOptions, - this._model); + this._model, + this._options.HasNamedVectors); return await this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.ScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false); } @@ -365,7 +369,8 @@ public async IAsyncEnumerable GetAsync(Expression> top, options, this.CollectionName, - this._model); + this._model, + this._options.HasNamedVectors); var results = await this.ExecuteQueryAsync(query, options.IncludeVectors, WeaviateConstants.ScorePropertyName, "GetAsync", cancellationToken).ConfigureAwait(false); await foreach (var record in results.Results.ConfigureAwait(false)) @@ -395,7 +400,8 @@ public async Task> HybridSearchAsync(TVect vectorProperty, textDataProperty, s_jsonSerializerOptions, - searchOptions); + searchOptions, + this._options.HasNamedVectors); return await this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.HybridScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false); } @@ -436,7 +442,7 @@ private async Task> ExecuteQueryAsync(string query, var mappedResults = collectionResults.Where(x => x is not null).Select(result => { - var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!, scorePropertyName); + var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!, scorePropertyName, this._options.HasNamedVectors); var record = VectorStoreErrorHandler.RunModelConversion( WeaviateConstants.VectorStoreSystemName, @@ -541,7 +547,11 @@ private IVectorStoreRecordMapper InitializeMapper() if (typeof(TRecord) == typeof(Dictionary)) { - var mapper = new WeaviateDynamicDataModelMapper(this.CollectionName, this._model, s_jsonSerializerOptions); + var mapper = new WeaviateDynamicDataModelMapper( + this.CollectionName, + this._options.HasNamedVectors, + this._model, + s_jsonSerializerOptions); return (mapper as IVectorStoreRecordMapper)!; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs index 5ed91595af5f..c0dfd17fe124 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs @@ -26,11 +26,10 @@ public static string BuildSearchQuery( JsonSerializerOptions jsonSerializerOptions, int top, VectorSearchOptions searchOptions, - VectorStoreRecordModel model) + VectorStoreRecordModel model, + bool hasNamedVectors) { - var vectorsQuery = searchOptions.IncludeVectors ? - $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" : - string.Empty; + var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); #pragma warning disable CS0618 // VectorSearchFilter is obsolete var filter = searchOptions switch @@ -52,7 +51,7 @@ public static string BuildSearchQuery( offset: {{searchOptions.Skip}} {{(filter is null ? "" : "where: " + filter)}} nearVector: { - targetVectors: ["{{vectorPropertyName}}"] + {{GetTargetVectorsQuery(hasNamedVectors, vectorPropertyName)}} vector: {{vectorArray}} } ) { @@ -77,11 +76,10 @@ public static string BuildQuery( int top, GetFilteredRecordOptions queryOptions, string collectionName, - VectorStoreRecordModel model) + VectorStoreRecordModel model, + bool hasNamedVectors) { - var vectorsQuery = queryOptions.IncludeVectors ? - $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" : - string.Empty; + var vectorsQuery = GetVectorsPropertyQuery(queryOptions.IncludeVectors, hasNamedVectors, model); var sortPaths = string.Join(",", queryOptions.OrderBy.Values.Select(sortInfo => { @@ -126,11 +124,10 @@ public static string BuildHybridSearchQuery( VectorStoreRecordVectorPropertyModel vectorProperty, VectorStoreRecordDataPropertyModel textProperty, JsonSerializerOptions jsonSerializerOptions, - HybridSearchOptions searchOptions) + HybridSearchOptions searchOptions, + bool hasNamedVectors) { - var vectorsQuery = searchOptions.IncludeVectors ? - $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" : - string.Empty; + var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); #pragma warning disable CS0618 // VectorSearchFilter is obsolete var filter = searchOptions switch @@ -154,7 +151,7 @@ public static string BuildHybridSearchQuery( hybrid: { query: "{{keywords}}" properties: ["{{textProperty.StorageName}}"] - targetVectors: ["{{vectorProperty.StorageName}}"] + {{GetTargetVectorsQuery(hasNamedVectors, vectorProperty.StorageName)}} vector: {{vectorArray}} fusionType: rankedFusion } @@ -173,6 +170,23 @@ public static string BuildHybridSearchQuery( #region private + private static string GetTargetVectorsQuery(bool hasNamedVectors, string vectorPropertyName) + { + return hasNamedVectors ? $"targetVectors: [\"{vectorPropertyName}\"]" : string.Empty; + } + + private static string GetVectorsPropertyQuery( + bool includeVectors, + bool hasNamedVectors, + VectorStoreRecordModel model) + { + return includeVectors + ? hasNamedVectors + ? $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" + : WeaviateConstants.ReservedSingleVectorPropertyName + : string.Empty; + } + #pragma warning disable CS0618 // Type or member is obsolete /// /// Builds filter for Weaviate search query. diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs index 668c81179afc..1c249758e9bd 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs @@ -82,7 +82,7 @@ public void MapFromDataToStorageModelMapsAllSupportedTypes() { // Arrange var key = new Guid("55555555-5555-5555-5555-555555555555"); - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); var dataModel = new Dictionary { @@ -184,7 +184,7 @@ public void MapFromDataToStorageModelMapsNullValues() ["NullableFloatVector"] = null }; - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); // Act var storageModel = sut.MapFromDataToStorageModel(dataModel); @@ -200,7 +200,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes() { // Arrange var key = new Guid("55555555-5555-5555-5555-555555555555"); - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); var storageModel = new JsonObject { @@ -308,7 +308,7 @@ public void MapFromStorageToDataModelMapsNullValues() } }; - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); // Act var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true }); @@ -324,7 +324,7 @@ public void MapFromStorageToDataModelMapsNullValues() public void MapFromStorageToDataModelThrowsForMissingKey() { // Arrange - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); var storageModel = new JsonObject(); @@ -353,7 +353,7 @@ public void MapFromDataToStorageModelSkipsMissingProperties() var key = new Guid("55555555-5555-5555-5555-555555555555"); var record = new Dictionary { ["Key"] = key }; - var sut = new WeaviateDynamicDataModelMapper("Collection", model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, model, s_jsonSerializerOptions); // Act var storageModel = sut.MapFromDataToStorageModel(record); @@ -383,7 +383,7 @@ public void MapFromStorageToDataModelSkipsMissingProperties() var key = new Guid("55555555-5555-5555-5555-555555555555"); - var sut = new WeaviateDynamicDataModelMapper("Collection", model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, model, s_jsonSerializerOptions); var storageModel = new JsonObject { @@ -398,4 +398,81 @@ public void MapFromStorageToDataModelSkipsMissingProperties() Assert.False(dataModel.ContainsKey("StringDataProp")); Assert.False(dataModel.ContainsKey("FloatVector")); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromDataToStorageModelMapsNamedVectorsCorrectly(bool hasNamedVectors) + { + // Arrange + var recordDefinition = new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(Guid)), + new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory)) + ] + }; + + var model = new WeaviateModelBuilder(hasNamedVectors).Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var record = new Dictionary { ["Key"] = key, ["FloatVector"] = new ReadOnlyMemory(s_floatVector) }; + var sut = new WeaviateDynamicDataModelMapper("Collection", hasNamedVectors, model, s_jsonSerializerOptions); + + // Act + var storageModel = sut.MapFromDataToStorageModel(record); + + // Assert + var vectorProperty = hasNamedVectors ? storageModel["vectors"]!["floatVector"] : storageModel["vector"]; + + Assert.Equal(key, (Guid?)storageModel["id"]); + Assert.Equal(s_floatVector, vectorProperty!.AsArray().GetValues().ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromStorageToDataModelMapsNamedVectorsCorrectly(bool hasNamedVectors) + { + // Arrange + var recordDefinition = new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(Guid)), + new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory)) + ] + }; + + var model = new WeaviateModelBuilder(hasNamedVectors).Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var sut = new WeaviateDynamicDataModelMapper("Collection", hasNamedVectors, model, s_jsonSerializerOptions); + + var storageModel = new JsonObject { ["id"] = key }; + + var vector = new JsonArray(s_floatVector.Select(l => (JsonValue)l).ToArray()); + + if (hasNamedVectors) + { + storageModel["vectors"] = new JsonObject + { + ["floatVector"] = vector + }; + } + else + { + storageModel["vector"] = vector; + } + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true }); + + // Assert + Assert.Equal(key, dataModel["Key"]); + Assert.Equal(s_floatVector, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs index acaf71858d43..b48d4cfcddd2 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -33,7 +33,7 @@ public void ItThrowsExceptionWithInvalidIndexKind() }); // Act & Assert - Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model)); + Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model)); } [Theory] @@ -56,7 +56,7 @@ public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string ex }); // Act - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model); + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); var actualIndexKind = schema.VectorConfigurations["Vector"].VectorIndexType; // Assert @@ -64,7 +64,7 @@ public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string ex } [Fact] - public void ItThrowsExceptionWithInvalidDistanceFunction() + public void ItThrowsExceptionWithUnsupportedDistanceFunction() { // Arrange var model = new WeaviateModelBuilder(HasNamedVectors) @@ -75,12 +75,12 @@ public void ItThrowsExceptionWithInvalidDistanceFunction() Properties = [ new VectorStoreRecordKeyProperty("Key", typeof(Guid)), - new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory)) { DistanceFunction = "non-existent-distance-function" } + new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory)) { DistanceFunction = "unsupported-distance-function" } ] }); // Act & Assert - Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model)); + Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model)); } [Theory] @@ -105,7 +105,7 @@ public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunct }); // Act - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model); + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); var actualDistanceFunction = schema.VectorConfigurations["Vector"].VectorIndexConfig?.Distance; @@ -178,7 +178,7 @@ public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyTy new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); // Act - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model); + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); var property = schema.Properties[0]; @@ -188,4 +188,48 @@ public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyTy Assert.True(property.IndexSearchable); Assert.True(property.IndexFilterable); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItReturnsCorrectSchemaWithValidVectorConfiguration(bool hasNamedVectors) + { + // Arrange + var model = new WeaviateModelBuilder(hasNamedVectors) + .Build( + typeof(Dictionary), + new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(Guid)), + new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory)) + { + DistanceFunction = DistanceFunction.CosineDistance, + IndexKind = IndexKind.Hnsw + } + ] + }); + + // Act + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", hasNamedVectors, model); + + // Assert + if (hasNamedVectors) + { + Assert.Null(schema.VectorIndexConfig?.Distance); + Assert.Null(schema.VectorIndexType); + Assert.True(schema.VectorConfigurations.ContainsKey("Vector")); + + Assert.Equal("cosine", schema.VectorConfigurations["Vector"].VectorIndexConfig?.Distance); + Assert.Equal("hnsw", schema.VectorConfigurations["Vector"].VectorIndexType); + } + else + { + Assert.False(schema.VectorConfigurations.ContainsKey("Vector")); + + Assert.Equal("cosine", schema.VectorIndexConfig?.Distance); + Assert.Equal("hnsw", schema.VectorIndexType); + } + } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs index 35a00c0376fc..3016b0083e2c 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs @@ -13,8 +13,10 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateVectorStoreCollectionSearchMappingTests { - [Fact] - public void MapSearchResultByDefaultReturnsValidResult() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapSearchResultByDefaultReturnsValidResult(bool hasNamedVectors) { // Arrange var jsonObject = new JsonObject @@ -22,11 +24,7 @@ public void MapSearchResultByDefaultReturnsValidResult() ["_additional"] = new JsonObject { ["distance"] = 0.5, - ["id"] = "55555555-5555-5555-5555-555555555555", - ["vectors"] = new JsonObject - { - ["descriptionEmbedding"] = new JsonArray(new List { 30, 31, 32, 33 }.Select(l => (JsonNode)l).ToArray()) - } + ["id"] = "55555555-5555-5555-5555-555555555555" }, ["description"] = "This is a great hotel.", ["hotelCode"] = 42, @@ -37,14 +35,27 @@ public void MapSearchResultByDefaultReturnsValidResult() ["timestamp"] = "2024-08-28T10:11:12-07:00" }; + var vector = new JsonArray(new List { 30, 31, 32, 33 }.Select(l => (JsonNode)l).ToArray()); + + if (hasNamedVectors) + { + jsonObject["_additional"]!["vectors"] = new JsonObject + { + ["descriptionEmbedding"] = vector + }; + } + else + { + jsonObject["_additional"]!["vector"] = vector; + } + // Act - var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(jsonObject, "distance"); + var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(jsonObject, "distance", hasNamedVectors); // Assert Assert.Equal(0.5, score); Assert.Equal("55555555-5555-5555-5555-555555555555", storageModel["id"]!.GetValue()); - Assert.Equal([30f, 31f, 32f, 33f], storageModel["vectors"]!["descriptionEmbedding"]!.AsArray().Select(l => l!.GetValue())); Assert.Equal("This is a great hotel.", storageModel["properties"]!["description"]!.GetValue()); Assert.Equal(42, storageModel["properties"]!["hotelCode"]!.GetValue()); Assert.Equal(4.5, storageModel["properties"]!["hotelRating"]!.GetValue()); @@ -52,5 +63,9 @@ public void MapSearchResultByDefaultReturnsValidResult() Assert.True(storageModel["properties"]!["parking_is_included"]!.GetValue()); Assert.Equal(["t1", "t2"], storageModel["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); Assert.Equal("2024-08-28T10:11:12-07:00", storageModel["properties"]!["timestamp"]!.GetValue()); + + var vectorProperty = hasNamedVectors ? storageModel["vectors"]!["descriptionEmbedding"] : storageModel["vector"]; + + Assert.Equal([30f, 31f, 32f, 33f], vectorProperty!.AsArray().Select(l => l!.GetValue())); } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs index c43d43cf9fcc..dc7272c7725a 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs @@ -49,8 +49,10 @@ public sealed class WeaviateVectorStoreRecordCollectionQueryBuilderTests private readonly ReadOnlyMemory _vector = new([31f, 32f, 33f, 34f]); - [Fact] - public void BuildSearchQueryByDefaultReturnsValidQuery() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BuildSearchQueryByDefaultReturnsValidQuery(bool hasNamedVectors) { // Arrange var expectedQuery = $$""" @@ -61,7 +63,7 @@ public void BuildSearchQueryByDefaultReturnsValidQuery() offset: 2 {{string.Empty}} nearVector: { - targetVectors: ["descriptionEmbedding"] + {{(hasNamedVectors ? "targetVectors: [\"descriptionEmbedding\"]" : string.Empty)}} vector: [31,32,33,34] } ) { @@ -89,7 +91,8 @@ HotelName HotelCode Tags s_jsonSerializerOptions, top: 3, searchOptions, - this._model); + this._model, + hasNamedVectors); // Assert Assert.Equal(expectedQuery, query); @@ -98,8 +101,10 @@ HotelName HotelCode Tags Assert.DoesNotContain("where", query); } - [Fact] - public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery(bool hasNamedVectors) { // Arrange var searchOptions = new VectorSearchOptions @@ -116,10 +121,13 @@ public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery() s_jsonSerializerOptions, top: 3, searchOptions, - this._model); + this._model, + hasNamedVectors); // Assert - Assert.Contains("vectors { DescriptionEmbedding }", query); + var vectorQuery = hasNamedVectors ? "vectors { DescriptionEmbedding }" : "vector"; + + Assert.Contains(vectorQuery, query); } [Fact] @@ -145,7 +153,8 @@ public void BuildSearchQueryWithFilterReturnsValidQuery() s_jsonSerializerOptions, top: 3, searchOptions, - this._model); + this._model, + hasNamedVectors: true); // Assert Assert.Contains(ExpectedFirstSubquery, query); @@ -170,7 +179,8 @@ public void BuildSearchQueryWithInvalidFilterValueThrowsException() s_jsonSerializerOptions, top: 3, searchOptions, - this._model)); + this._model, + hasNamedVectors: true)); } [Fact] @@ -191,7 +201,8 @@ public void BuildSearchQueryWithNonExistentPropertyInFilterThrowsException() s_jsonSerializerOptions, top: 3, searchOptions, - this._model)); + this._model, + hasNamedVectors: true)); } #region private diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs index 9d4b065d6c4c..4222608ab7c0 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.CRUD; -public class WeaviateBatchConformanceTests(WeaviateSimpleModelFixture fixture) - : BatchConformanceTests(fixture), IClassFixture +public class WeaviateBatchConformanceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : BatchConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateBatchConformanceTests_UnnamedVector(WeaviateSimpleModelUnnamedVectorFixture fixture) + : BatchConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs index 7e024a7a50bb..62825fac4ab1 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.CRUD; -public class WeaviateDynamicRecordConformanceTests(WeaviateDynamicDataModelFixture fixture) - : DynamicDataModelConformanceTests(fixture), IClassFixture +public class WeaviateDynamicRecordConformanceTests_NamedVectors(WeaviateDynamicDataModelNamedVectorsFixture fixture) + : DynamicDataModelConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateDynamicRecordConformanceTests_UnnamedVector(WeaviateDynamicDataModelUnnamedVectorFixture fixture) + : DynamicDataModelConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs index 3beb7b6e70e5..c2ad732eb59d 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.CRUD; -public class WeaviateRecordConformanceTests(WeaviateSimpleModelFixture fixture) - : RecordConformanceTests(fixture), IClassFixture +public class WeaviateRecordConformanceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateRecordConformanceTests_UnnamedVector(WeaviateSimpleModelUnnamedVectorFixture fixture) + : RecordConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs similarity index 52% rename from dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs rename to dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs index b78c7f705b59..3c817890ac16 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_NamedVectors.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.Collections; -public class WeaviateCollectionConformanceTests_NamedVectors(WeaviateNamedVectorsFixture fixture) - : CollectionConformanceTests(fixture), IClassFixture +public class WeaviateCollectionConformanceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateCollectionConformanceTests_UnnamedVector(WeaviateSimpleModelUnnamedVectorFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs deleted file mode 100644 index a070c49ed28c..000000000000 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests_UnnamedVector.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorDataSpecificationTests.Collections; -using WeaviateIntegrationTests.Support; -using Xunit; - -namespace WeaviateIntegrationTests.Collections; - -public class WeaviateCollectionConformanceTests_UnnamedVector(WeaviateUnnamedVectorFixture fixture) - : CollectionConformanceTests(fixture), IClassFixture -{ -} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs index b4dbf228ee37..f8f76dd27943 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs @@ -66,5 +66,7 @@ public override Task Equal_with_string_is_not_Contains() public new class Fixture : BasicFilterTests.Fixture { public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; } } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs similarity index 54% rename from dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs rename to dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs index a4c6db57a4c9..1b386273b6d6 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs @@ -7,6 +7,32 @@ namespace WeaviateIntegrationTests.HybridSearch; +public class WeaviateKeywordVectorizedHybridSearchTests_NamedVectors( + WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.VectorAndStringFixture vectorAndStringFixture, + WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.MultiTextFixture multiTextFixture) + : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + protected override string CollectionName => "VectorAndStringHybridSearch"; + } + + public new class MultiTextFixture : KeywordVectorizedHybridSearchComplianceTests.MultiTextFixture + { + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + protected override string CollectionName => "MultiTextHybridSearch"; + } +} + public class WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector( WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.VectorAndStringFixture vectorAndStringFixture, WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.MultiTextFixture multiTextFixture) diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs deleted file mode 100644 index b8ca5dfa0fe0..000000000000 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.cs +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorDataSpecificationTests.HybridSearch; -using VectorDataSpecificationTests.Support; -using WeaviateIntegrationTests.Support; -using Xunit; - -namespace WeaviateIntegrationTests.HybridSearch; - -public class WeaviateKeywordVectorizedHybridSearchTests_NamedVectors( - WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.VectorAndStringFixture vectorAndStringFixture, - WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.MultiTextFixture multiTextFixture) - : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), - IClassFixture, - IClassFixture -{ - public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture - { - public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; - - protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; - - protected override string CollectionName => "VectorAndStringHybridSearch"; - } - - public new class MultiTextFixture : KeywordVectorizedHybridSearchComplianceTests.MultiTextFixture - { - public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; - - protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; - - protected override string CollectionName => "MultiTextHybridSearch"; - } -} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs index 1745a902a348..831f05734d6b 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs @@ -8,7 +8,7 @@ namespace WeaviateIntegrationTests.Support.TestContainer; public sealed class WeaviateBuilder : ContainerBuilder { - public const string WeaviateImage = "semitechnologies/weaviate:1.26.4"; + public const string WeaviateImage = "semitechnologies/weaviate:1.28.12"; public const ushort WeaviateHttpPort = 8080; public const ushort WeaviateGrpcPort = 50051; diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs index 7e8ec36a04fa..038de8fa5fd2 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs @@ -10,5 +10,17 @@ public class WeaviateDynamicDataModelFixture : DynamicDataModelFixture // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - protected override string CollectionName => $"A{Guid.NewGuid():N}"; + protected override string CollectionName => this.GetUniqueCollectionName(); + + public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; +} + +public class WeaviateDynamicDataModelNamedVectorsFixture : WeaviateDynamicDataModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; +} + +public class WeaviateDynamicDataModelUnnamedVectorFixture : WeaviateDynamicDataModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs deleted file mode 100644 index a132cbed0ccb..000000000000 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateNamedVectorsFixture.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorDataSpecificationTests.Support; - -namespace WeaviateIntegrationTests.Support; - -public class WeaviateNamedVectorsFixture : VectorStoreFixture -{ - public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; - - // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. - // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; -} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs index 257eecec88ce..a10525cbb906 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs @@ -8,7 +8,21 @@ public class WeaviateSimpleModelFixture : SimpleModelFixture { public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - protected override string CollectionName => $"A{Guid.NewGuid():N}"; + protected override string CollectionName => this.GetUniqueCollectionName(); + + public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; +} + +public class WeaviateSimpleModelNamedVectorsFixture : WeaviateSimpleModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; +} + +public class WeaviateSimpleModelUnnamedVectorFixture : WeaviateSimpleModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs deleted file mode 100644 index 01cd29b92417..000000000000 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateUnnamedVectorFixture.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorDataSpecificationTests.Support; - -namespace WeaviateIntegrationTests.Support; - -public class WeaviateUnnamedVectorFixture : VectorStoreFixture -{ - public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; - - // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. - // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; -} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs new file mode 100644 index 000000000000..964a0292c1b2 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using WeaviateIntegrationTests.Support; +using VectorDataSpecificationTests.VectorSearch; +using Xunit; +using Microsoft.Extensions.VectorData; + +namespace WeaviateIntegrationTests.VectorSearch; + +public class WeaviateVectorSearchDistanceFunctionComplianceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + + /// + /// Tests vector search using , computing -(u · v) as a distance metric per Weaviate's convention. + /// Expects scores of -1 (exact match), 1 (opposite), and 0 (orthogonal), sorted ascending ([0, 2, 1]), with lower scores indicating closer matches. + /// . + /// + public override Task NegativeDotProductSimilarity() => this.SimpleSearch(DistanceFunction.NegativeDotProductSimilarity, -1, 1, 0, [0, 2, 1]); +} + +public class WeaviateVectorSearchDistanceFunctionComplianceTests_UnnamedVector(WeaviateDynamicDataModelNamedVectorsFixture fixture) + : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + + /// + /// Tests vector search using , computing -(u · v) as a distance metric per Weaviate's convention. + /// Expects scores of -1 (exact match), 1 (opposite), and 0 (orthogonal), sorted ascending ([0, 2, 1]), with lower scores indicating closer matches. + /// . + /// + public override Task NegativeDotProductSimilarity() => this.SimpleSearch(DistanceFunction.NegativeDotProductSimilarity, -1, 1, 0, [0, 2, 1]); +} From 17bb26c0b3e6c731e60d37e573535cc34e4e3774 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:08:33 -0700 Subject: [PATCH 12/15] Formatting fixes --- .../WeaviateVectorSearchDistanceFunctionComplianceTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs index 964a0292c1b2..0c6a5aadd390 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs @@ -1,9 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. -using WeaviateIntegrationTests.Support; +using Microsoft.Extensions.VectorData; using VectorDataSpecificationTests.VectorSearch; +using WeaviateIntegrationTests.Support; using Xunit; -using Microsoft.Extensions.VectorData; namespace WeaviateIntegrationTests.VectorSearch; From 8aeb9868fac91f40623de4dc66219d68d83eada6 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 11 Apr 2025 12:24:53 -0700 Subject: [PATCH 13/15] Fixes after merge --- .../WeaviateDynamicDataModelMapperTests.cs | 4 ++-- .../WeaviateVectorStoreCollectionCreateMappingTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs index 76891c205877..e15b354cc4ee 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs @@ -410,7 +410,7 @@ public void MapFromDataToStorageModelMapsNamedVectorsCorrectly(bool hasNamedVect Properties = [ new VectorStoreRecordKeyProperty("Key", typeof(Guid)), - new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory)) + new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory), 4) ] }; @@ -442,7 +442,7 @@ public void MapFromStorageToDataModelMapsNamedVectorsCorrectly(bool hasNamedVect Properties = [ new VectorStoreRecordKeyProperty("Key", typeof(Guid)), - new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory)) + new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory), 4) ] }; diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs index c9e1a368f061..20f0d560c8ef 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -203,7 +203,7 @@ public void ItReturnsCorrectSchemaWithValidVectorConfiguration(bool hasNamedVect Properties = [ new VectorStoreRecordKeyProperty("Key", typeof(Guid)), - new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory)) + new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory), 4) { DistanceFunction = DistanceFunction.CosineDistance, IndexKind = IndexKind.Hnsw From 925ec76c0486f1d3bc94704e3bcb0d4c17b6dbd1 Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Fri, 11 Apr 2025 12:28:56 -0700 Subject: [PATCH 14/15] Addressed PR feedback --- .../WeaviateDynamicDataModelMapper.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs index 506956eb62f3..524b305cc9c0 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs @@ -59,31 +59,31 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode Verify.NotNull(dataModel); // Transform generic data model to Weaviate object model. - var keyObject = JsonSerializer.SerializeToNode(dataModel[this._model.KeyProperty.ModelName]); + var keyNode = JsonSerializer.SerializeToNode(dataModel[this._model.KeyProperty.ModelName]); // Populate data properties. - var dataObject = new JsonObject(); + var dataNode = new JsonObject(); foreach (var property in this._model.DataProperties) { if (dataModel.TryGetValue(property.ModelName, out var dataValue)) { - dataObject[property.StorageName] = dataValue is null + dataNode[property.StorageName] = dataValue is null ? null : JsonSerializer.SerializeToNode(dataValue, property.Type, this._jsonSerializerOptions); } } // Populate vector properties. - JsonNode? vectorObject = null; + JsonNode? vectorNode = null; if (this._hasNamedVectors) { - vectorObject = new JsonObject(); + vectorNode = new JsonObject(); foreach (var property in this._model.VectorProperties) { if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) { - vectorObject[property.StorageName] = vectorValue is null + vectorNode[property.StorageName] = vectorValue is null ? null : JsonSerializer.SerializeToNode(vectorValue, property.Type, this._jsonSerializerOptions); } @@ -93,7 +93,7 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode { if (dataModel.TryGetValue(this._model.VectorProperty.ModelName, out var vectorValue)) { - vectorObject = vectorValue is null + vectorNode = vectorValue is null ? null : JsonSerializer.SerializeToNode(vectorValue, this._model.VectorProperty.Type, this._jsonSerializerOptions); } @@ -102,9 +102,9 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode return new JsonObject { { WeaviateConstants.CollectionPropertyName, JsonValue.Create(this._collectionName) }, - { WeaviateConstants.ReservedKeyPropertyName, keyObject }, - { WeaviateConstants.ReservedDataPropertyName, dataObject }, - { this._vectorPropertyName, vectorObject }, + { WeaviateConstants.ReservedKeyPropertyName, keyNode }, + { WeaviateConstants.ReservedDataPropertyName, dataNode }, + { this._vectorPropertyName, vectorNode }, }; } @@ -152,11 +152,11 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode } else { - var jsonObject = storageModel[WeaviateConstants.ReservedSingleVectorPropertyName]; + var jsonNode = storageModel[WeaviateConstants.ReservedSingleVectorPropertyName]; - if (jsonObject is not null) + if (jsonNode is not null) { - result.Add(this._model.VectorProperty.ModelName, jsonObject.Deserialize(this._model.VectorProperty.Type, this._jsonSerializerOptions)); + result.Add(this._model.VectorProperty.ModelName, jsonNode.Deserialize(this._model.VectorProperty.Type, this._jsonSerializerOptions)); } } } From f6def4be8c572e4cde17811e03fa94927c4532df Mon Sep 17 00:00:00 2001 From: Dmytro Struk <13853051+dmytrostruk@users.noreply.github.com> Date: Wed, 16 Apr 2025 07:25:31 -0700 Subject: [PATCH 15/15] Small fix after merge --- .../WeaviateVectorStoreRecordCollection.cs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index f95434063e99..997189a8731d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -141,14 +141,12 @@ public Task CreateCollectionAsync(CancellationToken cancellationToken = default) const string OperationName = "CreateCollectionSchema"; var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( - this.CollectionName, + this.Name, this._options.HasNamedVectors, this._model); return this.RunOperationAsync(OperationName, () => { - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(this.Name, this._model); - var request = new WeaviateCreateCollectionSchemaRequest(schema).Build(); return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken);