diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs index f31017ca8685..40012278a076 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/HttpV2/WeaviateGetCollectionsRequest.cs @@ -10,6 +10,6 @@ internal sealed class WeaviateGetCollectionsRequest public HttpRequestMessage Build() { - return HttpRequest.CreateGetRequest(ApiRoute, this); + return HttpRequest.CreateGetRequest(ApiRoute); } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs index e0f403ddb0e8..c6122eea8967 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchema.cs @@ -21,4 +21,13 @@ public WeaviateCollectionSchema(string collectionName) [JsonPropertyName("properties")] public List Properties { get; set; } = []; + + [JsonPropertyName("vectorizer")] + public string Vectorizer { get; set; } = WeaviateConstants.DefaultVectorizer; + + [JsonPropertyName("vectorIndexType")] + public string? VectorIndexType { get; set; } + + [JsonPropertyName("vectorIndexConfig")] + public WeaviateCollectionSchemaVectorIndexConfig? VectorIndexConfig { get; set; } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs index 75bd33471eb7..77830facd893 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/ModelV2/WeaviateCollectionSchemaVectorConfig.cs @@ -7,10 +7,8 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; internal sealed class WeaviateCollectionSchemaVectorConfig { - private const string DefaultVectorizer = "none"; - [JsonPropertyName("vectorizer")] - public Dictionary Vectorizer { get; set; } = new() { [DefaultVectorizer] = null }; + public Dictionary Vectorizer { get; set; } = new() { [WeaviateConstants.DefaultVectorizer] = null }; [JsonPropertyName("vectorIndexType")] public string? VectorIndexType { get; set; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs index f98b4ec35fde..f98d4a6304cd 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs @@ -16,6 +16,9 @@ internal sealed class WeaviateConstants /// Reserved vector property name in Weaviate. internal const string ReservedVectorPropertyName = "vectors"; + /// Reserved single vector property name in Weaviate. + internal const string ReservedSingleVectorPropertyName = "vector"; + /// Collection property name in Weaviate. internal const string CollectionPropertyName = "class"; @@ -27,4 +30,7 @@ internal sealed class WeaviateConstants /// Additional properties property name in Weaviate. internal const string AdditionalPropertiesPropertyName = "_additional"; + + /// Default vectorizer for vector properties in Weaviate. + internal const string DefaultVectorizer = "none"; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs index fe28c5c62e66..e80d34becf67 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateDynamicDataModelMapper.cs @@ -23,20 +23,33 @@ internal sealed class WeaviateDynamicDataModelMapper : IWeaviateMapperA for serialization/deserialization of record properties. private readonly JsonSerializerOptions _jsonSerializerOptions; + /// Gets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + private readonly bool _hasNamedVectors; + + /// Gets a vector property named used in Weaviate collection. + private readonly string _vectorPropertyName; + /// /// Initializes a new instance of the class. /// /// The name of the Weaviate collection + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection /// The model /// A for serialization/deserialization of record properties. public WeaviateDynamicDataModelMapper( string collectionName, + bool hasNamedVectors, VectorStoreRecordModel model, JsonSerializerOptions jsonSerializerOptions) { this._collectionName = collectionName; + this._hasNamedVectors = hasNamedVectors; this._model = model; this._jsonSerializerOptions = jsonSerializerOptions; + + this._vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; } public JsonObject MapFromDataToStorageModel(Dictionary dataModel) @@ -44,38 +57,52 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode Verify.NotNull(dataModel); // Transform generic data model to Weaviate object model. - var keyObject = JsonSerializer.SerializeToNode(dataModel[this._model.KeyProperty.ModelName]); + var keyNode = JsonSerializer.SerializeToNode(dataModel[this._model.KeyProperty.ModelName]); // Populate data properties. - var dataObject = new JsonObject(); + var dataNode = new JsonObject(); foreach (var property in this._model.DataProperties) { if (dataModel.TryGetValue(property.ModelName, out var dataValue)) { - dataObject[property.StorageName] = dataValue is null + dataNode[property.StorageName] = dataValue is null ? null : JsonSerializer.SerializeToNode(dataValue, property.Type, this._jsonSerializerOptions); } } // Populate vector properties. - var vectorObject = new JsonObject(); - foreach (var property in this._model.VectorProperties) + JsonNode? vectorNode = null; + + if (this._hasNamedVectors) { - if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + vectorNode = new JsonObject(); + foreach (var property in this._model.VectorProperties) { - vectorObject[property.StorageName] = vectorValue is null + if (dataModel.TryGetValue(property.ModelName, out var vectorValue)) + { + vectorNode[property.StorageName] = vectorValue is null + ? null + : JsonSerializer.SerializeToNode(vectorValue, property.Type, this._jsonSerializerOptions); + } + } + } + else + { + if (dataModel.TryGetValue(this._model.VectorProperty.ModelName, out var vectorValue)) + { + vectorNode = vectorValue is null ? null - : JsonSerializer.SerializeToNode(vectorValue, property.Type, this._jsonSerializerOptions); + : JsonSerializer.SerializeToNode(vectorValue, this._model.VectorProperty.Type, this._jsonSerializerOptions); } } return new JsonObject { { WeaviateConstants.CollectionPropertyName, JsonValue.Create(this._collectionName) }, - { WeaviateConstants.ReservedKeyPropertyName, keyObject }, - { WeaviateConstants.ReservedDataPropertyName, dataObject }, - { WeaviateConstants.ReservedVectorPropertyName, vectorObject }, + { WeaviateConstants.ReservedKeyPropertyName, keyNode }, + { WeaviateConstants.ReservedDataPropertyName, dataNode }, + { this._vectorPropertyName, vectorNode }, }; } @@ -109,13 +136,25 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode // Populate vector properties. if (options.IncludeVectors) { - foreach (var property in this._model.VectorProperties) + if (this._hasNamedVectors) + { + foreach (var property in this._model.VectorProperties) + { + var jsonObject = storageModel[WeaviateConstants.ReservedVectorPropertyName] as JsonObject; + + if (jsonObject is not null && jsonObject.TryGetPropertyValue(property.StorageName, out var vectorValue)) + { + result.Add(property.ModelName, vectorValue.Deserialize(property.Type, this._jsonSerializerOptions)); + } + } + } + else { - var jsonObject = storageModel[WeaviateConstants.ReservedVectorPropertyName] as JsonObject; + var jsonNode = storageModel[WeaviateConstants.ReservedSingleVectorPropertyName]; - if (jsonObject is not null && jsonObject.TryGetPropertyValue(property.StorageName, out var vectorValue)) + if (jsonNode is not null) { - result.Add(property.ModelName, vectorValue.Deserialize(property.Type, this._jsonSerializerOptions)); + result.Add(this._model.VectorProperty.ModelName, jsonNode.Deserialize(this._model.VectorProperty.Type, this._jsonSerializerOptions)); } } } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs index eaac5cc83f44..680d9ad3d40c 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateModelBuilder.cs @@ -6,37 +6,51 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; -internal class WeaviateModelBuilder() : VectorStoreRecordJsonModelBuilder(s_modelBuildingOptions) +internal class WeaviateModelBuilder(bool hasNamedVectors) : VectorStoreRecordJsonModelBuilder(GetModelBuildingOptions(hasNamedVectors)) { - private static readonly VectorStoreRecordModelBuildingOptions s_modelBuildingOptions = new() + private static VectorStoreRecordModelBuildingOptions GetModelBuildingOptions(bool hasNamedVectors) { - RequiresAtLeastOneVector = false, - SupportsMultipleKeys = false, - SupportsMultipleVectors = true, + return new() + { + RequiresAtLeastOneVector = false, + SupportsMultipleKeys = false, + SupportsMultipleVectors = hasNamedVectors, - SupportedKeyPropertyTypes = [typeof(Guid)], - SupportedDataPropertyTypes = s_supportedDataTypes, - SupportedEnumerableDataPropertyElementTypes = s_supportedDataTypes, - SupportedVectorPropertyTypes = s_supportedVectorTypes, + SupportedKeyPropertyTypes = [typeof(Guid)], + SupportedDataPropertyTypes = s_supportedDataTypes, + SupportedEnumerableDataPropertyElementTypes = s_supportedDataTypes, + SupportedVectorPropertyTypes = s_supportedVectorTypes, - UsesExternalSerializer = true, - ReservedKeyStorageName = WeaviateConstants.ReservedKeyPropertyName - }; + UsesExternalSerializer = true, + ReservedKeyStorageName = WeaviateConstants.ReservedKeyPropertyName + }; + } private static readonly HashSet s_supportedDataTypes = [ typeof(string), typeof(bool), + typeof(bool?), typeof(int), + typeof(int?), typeof(long), + typeof(long?), typeof(short), + typeof(short?), typeof(byte), + typeof(byte?), typeof(float), + typeof(float?), typeof(double), + typeof(double?), typeof(decimal), + typeof(decimal?), typeof(DateTime), + typeof(DateTime?), typeof(DateTimeOffset), + typeof(DateTimeOffset?), typeof(Guid), + typeof(Guid?) ]; internal static readonly HashSet s_supportedVectorTypes = diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs index 8b3ce214fa18..1e5a29bd541b 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStore.cs @@ -72,7 +72,8 @@ public IVectorStoreRecordCollection GetCollection( { VectorStoreRecordDefinition = vectorStoreRecordDefinition, Endpoint = this._options.Endpoint, - ApiKey = this._options.ApiKey + ApiKey = this._options.ApiKey, + HasNamedVectors = this._options.HasNamedVectors }) as IVectorStoreRecordCollection; return recordCollection; @@ -87,6 +88,9 @@ public async IAsyncEnumerable ListCollectionNamesAsync([EnumeratorCancel try { var httpResponse = await this._httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken).ConfigureAwait(false); + + httpResponse.EnsureSuccessStatusCode(); + var httpResponseContent = await httpResponse.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); collectionsResponse = JsonSerializer.Deserialize(httpResponseContent)!; diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs index 2448ceea8f18..852339436432 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionCreateMapping.cs @@ -19,9 +19,10 @@ internal static class WeaviateVectorStoreCollectionCreateMapping /// Maps record type properties to Weaviate collection schema for collection creation. /// /// The name of the vector store collection. + /// Gets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. /// The model. /// Weaviate collection schema. - public static WeaviateCollectionSchema MapToSchema(string collectionName, VectorStoreRecordModel model) + public static WeaviateCollectionSchema MapToSchema(string collectionName, bool hasNamedVectors, VectorStoreRecordModel model) { var schema = new WeaviateCollectionSchema(collectionName); @@ -38,16 +39,28 @@ public static WeaviateCollectionSchema MapToSchema(string collectionName, Vector } // Handle vector properties. - foreach (var property in model.VectorProperties) + if (hasNamedVectors) { - schema.VectorConfigurations.Add(property.StorageName, new WeaviateCollectionSchemaVectorConfig + foreach (var property in model.VectorProperties) { - VectorIndexType = MapIndexKind(property.IndexKind, property.StorageName), - VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + schema.VectorConfigurations.Add(property.StorageName, new WeaviateCollectionSchemaVectorConfig { - Distance = MapDistanceFunction(property.DistanceFunction, property.StorageName) - } - }); + VectorIndexType = MapIndexKind(property.IndexKind, property.StorageName), + VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(property.DistanceFunction, property.StorageName) + } + }); + } + } + else + { + var vectorProperty = model.VectorProperty; + schema.VectorIndexType = MapIndexKind(vectorProperty.IndexKind, vectorProperty.StorageName); + schema.VectorIndexConfig = new WeaviateCollectionSchemaVectorIndexConfig + { + Distance = MapDistanceFunction(vectorProperty.DistanceFunction, vectorProperty.StorageName) + }; } return schema; @@ -110,7 +123,7 @@ private static string MapDistanceFunction(string? distanceFunction, string vecto DistanceFunction.EuclideanSquaredDistance => EuclideanSquared, DistanceFunction.Hamming => Hamming, DistanceFunction.ManhattanDistance => Manhattan, - _ => throw new InvalidOperationException( + _ => throw new NotSupportedException( $"Distance function '{distanceFunction}' on {nameof(VectorStoreRecordVectorProperty)} '{vectorPropertyName}' is not supported by the Weaviate VectorStore. " + $"Supported distance functions: {string.Join(", ", DistanceFunction.CosineDistance, diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs index 3842a3aded97..02ec36be81a1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs @@ -13,7 +13,10 @@ internal static class WeaviateVectorStoreCollectionSearchMapping /// /// Maps vector search result to the format, which is processable by . /// - public static (JsonObject StorageModel, double? Score) MapSearchResult(JsonNode result, string scorePropertyName) + public static (JsonObject StorageModel, double? Score) MapSearchResult( + JsonNode result, + string scorePropertyName, + bool hasNamedVectors) { var additionalProperties = result[WeaviateConstants.AdditionalPropertiesPropertyName]; @@ -25,14 +28,18 @@ public static (JsonObject StorageModel, double? Score) MapSearchResult(JsonNode _ => null }; + var vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; + var id = additionalProperties?[WeaviateConstants.ReservedKeyPropertyName]; - var vectors = additionalProperties?[WeaviateConstants.ReservedVectorPropertyName]; + var vectors = additionalProperties?[vectorPropertyName]; var storageModel = new JsonObject { { WeaviateConstants.ReservedKeyPropertyName, id?.DeepClone() }, { WeaviateConstants.ReservedDataPropertyName, result?.DeepClone() }, - { WeaviateConstants.ReservedVectorPropertyName, vectors?.DeepClone() }, + { vectorPropertyName, vectors?.DeepClone() }, }; return (storageModel, score); diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs index fb9cdc208e57..9d955f8d48a6 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreOptions.cs @@ -27,4 +27,11 @@ public sealed class WeaviateVectorStoreOptions /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. /// public string? ApiKey { get; set; } = null; + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// Defaults to multiple named vectors. + /// . + /// + public bool HasNamedVectors { get; set; } = true; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs index 7275b1b5e08b..997189a8731d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs @@ -103,12 +103,13 @@ public WeaviateVectorStoreRecordCollection( this.Name = name; this._options = options ?? new(); this._apiKey = this._options.ApiKey; - this._model = new WeaviateModelBuilder().Build(typeof(TRecord), this._options.VectorStoreRecordDefinition, s_jsonSerializerOptions); + this._model = new WeaviateModelBuilder(this._options.HasNamedVectors) + .Build(typeof(TRecord), this._options.VectorStoreRecordDefinition, s_jsonSerializerOptions); // Assign mapper. this._mapper = typeof(TRecord) == typeof(Dictionary) - ? (new WeaviateDynamicDataModelMapper(this.Name, this._model, s_jsonSerializerOptions) as IWeaviateMapper)! - : new WeaviateVectorStoreRecordMapper(this.Name, this._model, s_jsonSerializerOptions); + ? (new WeaviateDynamicDataModelMapper(this.Name, this._options.HasNamedVectors, this._model, s_jsonSerializerOptions) as IWeaviateMapper)! + : new WeaviateVectorStoreRecordMapper(this.Name, this._options.HasNamedVectors, this._model, s_jsonSerializerOptions); this._collectionMetadata = new() { @@ -139,13 +140,16 @@ public Task CreateCollectionAsync(CancellationToken cancellationToken = default) { const string OperationName = "CreateCollectionSchema"; + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema( + this.Name, + this._options.HasNamedVectors, + this._model); + return this.RunOperationAsync(OperationName, () => { - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(this.Name, this._model); - var request = new WeaviateCreateCollectionSchemaRequest(schema).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -167,7 +171,7 @@ public Task DeleteCollectionAsync(CancellationToken cancellationToken = default) { var request = new WeaviateDeleteCollectionSchemaRequest(this.Name).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -187,7 +191,7 @@ public Task DeleteAsync(TKey key, CancellationToken cancellationToken = default) var request = new WeaviateDeleteObjectRequest(this.Name, guid).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -221,7 +225,7 @@ public Task DeleteAsync(IEnumerable keys, CancellationToken cancellationTo var request = new WeaviateDeleteObjectBatchRequest(match).Build(); - return this.ExecuteRequestAsync(request, cancellationToken); + return this.ExecuteRequestAsync(request, cancellationToken: cancellationToken); }); } @@ -351,7 +355,8 @@ public IAsyncEnumerable> VectorizedSearchAsync GetAsync(Expression> filter top, options, this.Name, - this._model); + this._model, + this._options.HasNamedVectors); return this.ExecuteQueryAsync(query, options.IncludeVectors, WeaviateConstants.ScorePropertyName, "GetAsync", cancellationToken) .SelectAsync(result => result.Record, cancellationToken: cancellationToken); @@ -397,7 +403,8 @@ public IAsyncEnumerable> HybridSearchAsync( vectorProperty, textDataProperty, s_jsonSerializerOptions, - searchOptions); + searchOptions, + this._options.HasNamedVectors); return this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.HybridScorePropertyName, OperationName, cancellationToken); } @@ -440,7 +447,7 @@ private async IAsyncEnumerable> ExecuteQueryAsync(st { if (result is not null) { - var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result, scorePropertyName); + var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result, scorePropertyName, this._options.HasNamedVectors); var record = VectorStoreErrorHandler.RunModelConversion( WeaviateConstants.VectorStoreSystemName, @@ -454,7 +461,10 @@ private async IAsyncEnumerable> ExecuteQueryAsync(st } } - private Task ExecuteRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken) + private async Task ExecuteRequestAsync( + HttpRequestMessage request, + bool ensureSuccessStatusCode = true, + CancellationToken cancellationToken = default) { request.RequestUri = new Uri(this._endpoint, request.RequestUri!); @@ -463,12 +473,21 @@ private Task ExecuteRequestAsync(HttpRequestMessage request request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", this._apiKey); } - return this._httpClient.SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken); + var response = await this._httpClient + .SendAsync(request, HttpCompletionOption.ResponseContentRead, cancellationToken) + .ConfigureAwait(false); + + if (ensureSuccessStatusCode) + { + response.EnsureSuccessStatusCode(); + } + + return response; } private async Task<(TResponse?, string)> ExecuteRequestWithResponseContentAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var response = await this.ExecuteRequestAsync(request, cancellationToken).ConfigureAwait(false); + var response = await this.ExecuteRequestAsync(request, cancellationToken: cancellationToken).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); @@ -486,12 +505,15 @@ private Task ExecuteRequestAsync(HttpRequestMessage request private async Task ExecuteRequestWithNotFoundHandlingAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - var response = await this.ExecuteRequestAsync(request, cancellationToken).ConfigureAwait(false); + var response = await this.ExecuteRequestAsync(request, ensureSuccessStatusCode: false, cancellationToken: cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.NotFound) { return default; } + response.EnsureSuccessStatusCode(); + var responseContent = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); var responseModel = JsonSerializer.Deserialize(responseContent, s_jsonSerializerOptions); diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs index f758d77499a2..60becffbaad1 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionOptions.cs @@ -39,4 +39,11 @@ public sealed class WeaviateVectorStoreRecordCollectionOptions /// This parameter is optional because authentication may be disabled in local clusters for testing purposes. /// public string? ApiKey { get; set; } = null; + + /// + /// Gets or sets a value indicating whether the vectors in the store are named and multiple vectors are supported, or whether there is just a single unnamed vector in Weaviate collection. + /// Defaults to multiple named vectors. + /// . + /// + public bool HasNamedVectors { get; set; } = true; } diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs index 5ed91595af5f..c0dfd17fe124 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollectionQueryBuilder.cs @@ -26,11 +26,10 @@ public static string BuildSearchQuery( JsonSerializerOptions jsonSerializerOptions, int top, VectorSearchOptions searchOptions, - VectorStoreRecordModel model) + VectorStoreRecordModel model, + bool hasNamedVectors) { - var vectorsQuery = searchOptions.IncludeVectors ? - $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" : - string.Empty; + var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); #pragma warning disable CS0618 // VectorSearchFilter is obsolete var filter = searchOptions switch @@ -52,7 +51,7 @@ public static string BuildSearchQuery( offset: {{searchOptions.Skip}} {{(filter is null ? "" : "where: " + filter)}} nearVector: { - targetVectors: ["{{vectorPropertyName}}"] + {{GetTargetVectorsQuery(hasNamedVectors, vectorPropertyName)}} vector: {{vectorArray}} } ) { @@ -77,11 +76,10 @@ public static string BuildQuery( int top, GetFilteredRecordOptions queryOptions, string collectionName, - VectorStoreRecordModel model) + VectorStoreRecordModel model, + bool hasNamedVectors) { - var vectorsQuery = queryOptions.IncludeVectors ? - $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" : - string.Empty; + var vectorsQuery = GetVectorsPropertyQuery(queryOptions.IncludeVectors, hasNamedVectors, model); var sortPaths = string.Join(",", queryOptions.OrderBy.Values.Select(sortInfo => { @@ -126,11 +124,10 @@ public static string BuildHybridSearchQuery( VectorStoreRecordVectorPropertyModel vectorProperty, VectorStoreRecordDataPropertyModel textProperty, JsonSerializerOptions jsonSerializerOptions, - HybridSearchOptions searchOptions) + HybridSearchOptions searchOptions, + bool hasNamedVectors) { - var vectorsQuery = searchOptions.IncludeVectors ? - $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" : - string.Empty; + var vectorsQuery = GetVectorsPropertyQuery(searchOptions.IncludeVectors, hasNamedVectors, model); #pragma warning disable CS0618 // VectorSearchFilter is obsolete var filter = searchOptions switch @@ -154,7 +151,7 @@ public static string BuildHybridSearchQuery( hybrid: { query: "{{keywords}}" properties: ["{{textProperty.StorageName}}"] - targetVectors: ["{{vectorProperty.StorageName}}"] + {{GetTargetVectorsQuery(hasNamedVectors, vectorProperty.StorageName)}} vector: {{vectorArray}} fusionType: rankedFusion } @@ -173,6 +170,23 @@ public static string BuildHybridSearchQuery( #region private + private static string GetTargetVectorsQuery(bool hasNamedVectors, string vectorPropertyName) + { + return hasNamedVectors ? $"targetVectors: [\"{vectorPropertyName}\"]" : string.Empty; + } + + private static string GetVectorsPropertyQuery( + bool includeVectors, + bool hasNamedVectors, + VectorStoreRecordModel model) + { + return includeVectors + ? hasNamedVectors + ? $"vectors {{ {string.Join(" ", model.VectorProperties.Select(p => p.StorageName))} }}" + : WeaviateConstants.ReservedSingleVectorPropertyName + : string.Empty; + } + #pragma warning disable CS0618 // Type or member is obsolete /// /// Builds filter for Weaviate search query. diff --git a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs index ce5a38ebc0e2..1ecf80ad651d 100644 --- a/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs +++ b/dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordMapper.cs @@ -13,17 +13,26 @@ internal sealed class WeaviateVectorStoreRecordMapper : IWeaviateMapper #pragma warning restore CS0618 { private readonly string _collectionName; + private readonly bool _hasNamedVectors; private readonly VectorStoreRecordModel _model; private readonly JsonSerializerOptions _jsonSerializerOptions; + private readonly string _vectorPropertyName; + public WeaviateVectorStoreRecordMapper( string collectionName, + bool hasNamedVectors, VectorStoreRecordModel model, JsonSerializerOptions jsonSerializerOptions) { this._collectionName = collectionName; + this._hasNamedVectors = hasNamedVectors; this._model = model; this._jsonSerializerOptions = jsonSerializerOptions; + + this._vectorPropertyName = hasNamedVectors ? + WeaviateConstants.ReservedVectorPropertyName : + WeaviateConstants.ReservedSingleVectorPropertyName; } public JsonObject MapFromDataToStorageModel(TRecord dataModel) @@ -41,7 +50,7 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel) // account e.g. naming policies. TemporaryStorageName gets populated in the model builder - containing that name - once VectorStoreModelBuildingOptions.ReservedKeyPropertyName is set { WeaviateConstants.ReservedKeyPropertyName, jsonNodeDataModel[this._model.KeyProperty.TemporaryStorageName!]!.DeepClone() }, { WeaviateConstants.ReservedDataPropertyName, new JsonObject() }, - { WeaviateConstants.ReservedVectorPropertyName, new JsonObject() }, + { this._vectorPropertyName, new JsonObject() }, }; // Populate data properties. @@ -56,13 +65,26 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel) } // Populate vector properties. - foreach (var property in this._model.VectorProperties) + if (this._hasNamedVectors) { - var node = jsonNodeDataModel[property.StorageName]; + foreach (var property in this._model.VectorProperties) + { + var node = jsonNodeDataModel[property.StorageName]; + + if (node is not null) + { + weaviateObjectModel[this._vectorPropertyName]![property.StorageName] = node.DeepClone(); + } + } + } + else + { + var vectorProperty = this._model.VectorProperty; + var node = jsonNodeDataModel[vectorProperty.StorageName]; if (node is not null) { - weaviateObjectModel[WeaviateConstants.ReservedVectorPropertyName]![property.StorageName] = node.DeepClone(); + weaviateObjectModel[this._vectorPropertyName] = node.DeepClone(); } } @@ -97,13 +119,26 @@ public TRecord MapFromStorageToDataModel(JsonObject storageModel, StorageToDataM // Populate vector properties. if (options.IncludeVectors) { - foreach (var property in this._model.VectorProperties) + if (this._hasNamedVectors) + { + foreach (var property in this._model.VectorProperties) + { + var node = storageModel[this._vectorPropertyName]?[property.StorageName]; + + if (node is not null) + { + jsonNodeDataModel[property.StorageName] = node.DeepClone(); + } + } + } + else { - var node = storageModel[WeaviateConstants.ReservedVectorPropertyName]?[property.StorageName]; + var vectorProperty = this._model.VectorProperty; + var node = storageModel[this._vectorPropertyName]; if (node is not null) { - jsonNodeDataModel[property.StorageName] = node.DeepClone(); + jsonNodeDataModel[vectorProperty.StorageName] = node.DeepClone(); } } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs index e36990bc3630..e15b354cc4ee 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateDynamicDataModelMapperTests.cs @@ -18,6 +18,8 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateDynamicDataModelMapperTests { + private const bool HasNamedVectors = true; + private static readonly JsonSerializerOptions s_jsonSerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase, @@ -29,7 +31,7 @@ public sealed class WeaviateDynamicDataModelMapperTests } }; - private static readonly VectorStoreRecordModel s_model = new WeaviateModelBuilder() + private static readonly VectorStoreRecordModel s_model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(Dictionary), new VectorStoreRecordDefinition @@ -80,7 +82,7 @@ public void MapFromDataToStorageModelMapsAllSupportedTypes() { // Arrange var key = new Guid("55555555-5555-5555-5555-555555555555"); - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); var dataModel = new Dictionary { @@ -182,7 +184,7 @@ public void MapFromDataToStorageModelMapsNullValues() ["NullableFloatVector"] = null }; - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); // Act var storageModel = sut.MapFromDataToStorageModel(dataModel); @@ -198,7 +200,7 @@ public void MapFromStorageToDataModelMapsAllSupportedTypes() { // Arrange var key = new Guid("55555555-5555-5555-5555-555555555555"); - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); var storageModel = new JsonObject { @@ -306,7 +308,7 @@ public void MapFromStorageToDataModelMapsNullValues() } }; - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); // Act var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true }); @@ -322,7 +324,7 @@ public void MapFromStorageToDataModelMapsNullValues() public void MapFromStorageToDataModelThrowsForMissingKey() { // Arrange - var sut = new WeaviateDynamicDataModelMapper("Collection", s_model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, s_model, s_jsonSerializerOptions); var storageModel = new JsonObject(); @@ -346,12 +348,12 @@ public void MapFromDataToStorageModelSkipsMissingProperties() ] }; - var model = new WeaviateModelBuilder().Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); + var model = new WeaviateModelBuilder(HasNamedVectors).Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); var key = new Guid("55555555-5555-5555-5555-555555555555"); var record = new Dictionary { ["Key"] = key }; - var sut = new WeaviateDynamicDataModelMapper("Collection", model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, model, s_jsonSerializerOptions); // Act var storageModel = sut.MapFromDataToStorageModel(record); @@ -377,11 +379,11 @@ public void MapFromStorageToDataModelSkipsMissingProperties() ] }; - var model = new WeaviateModelBuilder().Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); + var model = new WeaviateModelBuilder(HasNamedVectors).Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); var key = new Guid("55555555-5555-5555-5555-555555555555"); - var sut = new WeaviateDynamicDataModelMapper("Collection", model, s_jsonSerializerOptions); + var sut = new WeaviateDynamicDataModelMapper("Collection", HasNamedVectors, model, s_jsonSerializerOptions); var storageModel = new JsonObject { @@ -396,4 +398,81 @@ public void MapFromStorageToDataModelSkipsMissingProperties() Assert.False(dataModel.ContainsKey("StringDataProp")); Assert.False(dataModel.ContainsKey("FloatVector")); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromDataToStorageModelMapsNamedVectorsCorrectly(bool hasNamedVectors) + { + // Arrange + var recordDefinition = new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(Guid)), + new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory), 4) + ] + }; + + var model = new WeaviateModelBuilder(hasNamedVectors).Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var record = new Dictionary { ["Key"] = key, ["FloatVector"] = new ReadOnlyMemory(s_floatVector) }; + var sut = new WeaviateDynamicDataModelMapper("Collection", hasNamedVectors, model, s_jsonSerializerOptions); + + // Act + var storageModel = sut.MapFromDataToStorageModel(record); + + // Assert + var vectorProperty = hasNamedVectors ? storageModel["vectors"]!["floatVector"] : storageModel["vector"]; + + Assert.Equal(key, (Guid?)storageModel["id"]); + Assert.Equal(s_floatVector, vectorProperty!.AsArray().GetValues().ToArray()); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromStorageToDataModelMapsNamedVectorsCorrectly(bool hasNamedVectors) + { + // Arrange + var recordDefinition = new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(Guid)), + new VectorStoreRecordVectorProperty("FloatVector", typeof(ReadOnlyMemory), 4) + ] + }; + + var model = new WeaviateModelBuilder(hasNamedVectors).Build(typeof(Dictionary), recordDefinition, s_jsonSerializerOptions); + + var key = new Guid("55555555-5555-5555-5555-555555555555"); + + var sut = new WeaviateDynamicDataModelMapper("Collection", hasNamedVectors, model, s_jsonSerializerOptions); + + var storageModel = new JsonObject { ["id"] = key }; + + var vector = new JsonArray(s_floatVector.Select(l => (JsonValue)l).ToArray()); + + if (hasNamedVectors) + { + storageModel["vectors"] = new JsonObject + { + ["floatVector"] = vector + }; + } + else + { + storageModel["vector"] = vector; + } + + // Act + var dataModel = sut.MapFromStorageToDataModel(storageModel, new StorageToDataModelMapperOptions { IncludeVectors = true }); + + // Assert + Assert.Equal(key, dataModel["Key"]); + Assert.Equal(s_floatVector, ((ReadOnlyMemory)dataModel["FloatVector"]!).ToArray()); + } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs index 23570ac0bf5b..20f0d560c8ef 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionCreateMappingTests.cs @@ -14,11 +14,13 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateVectorStoreCollectionCreateMappingTests { + private const bool HasNamedVectors = true; + [Fact] public void ItThrowsExceptionWithInvalidIndexKind() { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(Dictionary), new VectorStoreRecordDefinition @@ -31,7 +33,7 @@ public void ItThrowsExceptionWithInvalidIndexKind() }); // Act & Assert - Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model)); + Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model)); } [Theory] @@ -41,7 +43,7 @@ public void ItThrowsExceptionWithInvalidIndexKind() public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string expectedIndexKind) { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(Dictionary), new VectorStoreRecordDefinition @@ -54,7 +56,7 @@ public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string ex }); // Act - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model); + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); var actualIndexKind = schema.VectorConfigurations["Vector"].VectorIndexType; // Assert @@ -62,10 +64,10 @@ public void ItReturnsCorrectSchemaWithValidIndexKind(string indexKind, string ex } [Fact] - public void ItThrowsExceptionWithInvalidDistanceFunction() + public void ItThrowsExceptionWithUnsupportedDistanceFunction() { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(Dictionary), new VectorStoreRecordDefinition @@ -73,12 +75,12 @@ public void ItThrowsExceptionWithInvalidDistanceFunction() Properties = [ new VectorStoreRecordKeyProperty("Key", typeof(Guid)), - new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { DistanceFunction = "non-existent-distance-function" } + new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory), 10) { DistanceFunction = "unsupported-distance-function" } ] }); // Act & Assert - Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model)); + Assert.Throws(() => WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model)); } [Theory] @@ -90,7 +92,7 @@ public void ItThrowsExceptionWithInvalidDistanceFunction() public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunction, string expectedDistanceFunction) { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(Dictionary), new VectorStoreRecordDefinition @@ -103,7 +105,7 @@ public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunct }); // Act - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model); + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); var actualDistanceFunction = schema.VectorConfigurations["Vector"].VectorIndexConfig?.Distance; @@ -158,12 +160,10 @@ public void ItReturnsCorrectSchemaWithValidDistanceFunction(string distanceFunct [InlineData(typeof(bool?), "boolean")] [InlineData(typeof(List), "boolean[]")] [InlineData(typeof(List), "boolean[]")] - [InlineData(typeof(object), "object")] - [InlineData(typeof(List), "object[]")] public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyType) { // Arrange - var model = new WeaviateModelBuilder() + var model = new WeaviateModelBuilder(HasNamedVectors) .Build( typeof(Dictionary), new VectorStoreRecordDefinition @@ -178,7 +178,7 @@ public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyTy new JsonSerializerOptions { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }); // Act - var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", model); + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", HasNamedVectors, model); var property = schema.Properties[0]; @@ -188,4 +188,48 @@ public void ItMapsPropertyCorrectly(Type propertyType, string expectedPropertyTy Assert.True(property.IndexSearchable); Assert.True(property.IndexFilterable); } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void ItReturnsCorrectSchemaWithValidVectorConfiguration(bool hasNamedVectors) + { + // Arrange + var model = new WeaviateModelBuilder(hasNamedVectors) + .Build( + typeof(Dictionary), + new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("Key", typeof(Guid)), + new VectorStoreRecordVectorProperty("Vector", typeof(ReadOnlyMemory), 4) + { + DistanceFunction = DistanceFunction.CosineDistance, + IndexKind = IndexKind.Hnsw + } + ] + }); + + // Act + var schema = WeaviateVectorStoreCollectionCreateMapping.MapToSchema(collectionName: "CollectionName", hasNamedVectors, model); + + // Assert + if (hasNamedVectors) + { + Assert.Null(schema.VectorIndexConfig?.Distance); + Assert.Null(schema.VectorIndexType); + Assert.True(schema.VectorConfigurations.ContainsKey("Vector")); + + Assert.Equal("cosine", schema.VectorConfigurations["Vector"].VectorIndexConfig?.Distance); + Assert.Equal("hnsw", schema.VectorConfigurations["Vector"].VectorIndexType); + } + else + { + Assert.False(schema.VectorConfigurations.ContainsKey("Vector")); + + Assert.Equal("cosine", schema.VectorIndexConfig?.Distance); + Assert.Equal("hnsw", schema.VectorIndexType); + } + } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs index 35a00c0376fc..3016b0083e2c 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreCollectionSearchMappingTests.cs @@ -13,8 +13,10 @@ namespace SemanticKernel.Connectors.Weaviate.UnitTests; /// public sealed class WeaviateVectorStoreCollectionSearchMappingTests { - [Fact] - public void MapSearchResultByDefaultReturnsValidResult() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapSearchResultByDefaultReturnsValidResult(bool hasNamedVectors) { // Arrange var jsonObject = new JsonObject @@ -22,11 +24,7 @@ public void MapSearchResultByDefaultReturnsValidResult() ["_additional"] = new JsonObject { ["distance"] = 0.5, - ["id"] = "55555555-5555-5555-5555-555555555555", - ["vectors"] = new JsonObject - { - ["descriptionEmbedding"] = new JsonArray(new List { 30, 31, 32, 33 }.Select(l => (JsonNode)l).ToArray()) - } + ["id"] = "55555555-5555-5555-5555-555555555555" }, ["description"] = "This is a great hotel.", ["hotelCode"] = 42, @@ -37,14 +35,27 @@ public void MapSearchResultByDefaultReturnsValidResult() ["timestamp"] = "2024-08-28T10:11:12-07:00" }; + var vector = new JsonArray(new List { 30, 31, 32, 33 }.Select(l => (JsonNode)l).ToArray()); + + if (hasNamedVectors) + { + jsonObject["_additional"]!["vectors"] = new JsonObject + { + ["descriptionEmbedding"] = vector + }; + } + else + { + jsonObject["_additional"]!["vector"] = vector; + } + // Act - var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(jsonObject, "distance"); + var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(jsonObject, "distance", hasNamedVectors); // Assert Assert.Equal(0.5, score); Assert.Equal("55555555-5555-5555-5555-555555555555", storageModel["id"]!.GetValue()); - Assert.Equal([30f, 31f, 32f, 33f], storageModel["vectors"]!["descriptionEmbedding"]!.AsArray().Select(l => l!.GetValue())); Assert.Equal("This is a great hotel.", storageModel["properties"]!["description"]!.GetValue()); Assert.Equal(42, storageModel["properties"]!["hotelCode"]!.GetValue()); Assert.Equal(4.5, storageModel["properties"]!["hotelRating"]!.GetValue()); @@ -52,5 +63,9 @@ public void MapSearchResultByDefaultReturnsValidResult() Assert.True(storageModel["properties"]!["parking_is_included"]!.GetValue()); Assert.Equal(["t1", "t2"], storageModel["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); Assert.Equal("2024-08-28T10:11:12-07:00", storageModel["properties"]!["timestamp"]!.GetValue()); + + var vectorProperty = hasNamedVectors ? storageModel["vectors"]!["descriptionEmbedding"] : storageModel["vector"]; + + Assert.Equal([30f, 31f, 32f, 33f], vectorProperty!.AsArray().Select(l => l!.GetValue())); } } diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs index b76aacfb281e..8bbdf51404e3 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordCollectionQueryBuilderTests.cs @@ -32,7 +32,7 @@ public sealed class WeaviateVectorStoreRecordCollectionQueryBuilderTests } }; - private readonly VectorStoreRecordModel _model = new WeaviateModelBuilder() + private readonly VectorStoreRecordModel _model = new WeaviateModelBuilder(hasNamedVectors: true) .Build( typeof(Dictionary), new() @@ -49,8 +49,10 @@ public sealed class WeaviateVectorStoreRecordCollectionQueryBuilderTests private readonly ReadOnlyMemory _vector = new([31f, 32f, 33f, 34f]); - [Fact] - public void BuildSearchQueryByDefaultReturnsValidQuery() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BuildSearchQueryByDefaultReturnsValidQuery(bool hasNamedVectors) { // Arrange var expectedQuery = $$""" @@ -61,7 +63,7 @@ public void BuildSearchQueryByDefaultReturnsValidQuery() offset: 2 {{string.Empty}} nearVector: { - targetVectors: ["descriptionEmbedding"] + {{(hasNamedVectors ? "targetVectors: [\"descriptionEmbedding\"]" : string.Empty)}} vector: [31,32,33,34] } ) { @@ -89,7 +91,8 @@ HotelName HotelCode Tags s_jsonSerializerOptions, top: 3, searchOptions, - this._model); + this._model, + hasNamedVectors); // Assert Assert.Equal(expectedQuery, query); @@ -98,8 +101,10 @@ HotelName HotelCode Tags Assert.DoesNotContain("where", query); } - [Fact] - public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery(bool hasNamedVectors) { // Arrange var searchOptions = new VectorSearchOptions @@ -116,10 +121,13 @@ public void BuildSearchQueryWithIncludedVectorsReturnsValidQuery() s_jsonSerializerOptions, top: 3, searchOptions, - this._model); + this._model, + hasNamedVectors); // Assert - Assert.Contains("vectors { DescriptionEmbedding }", query); + var vectorQuery = hasNamedVectors ? "vectors { DescriptionEmbedding }" : "vector"; + + Assert.Contains(vectorQuery, query); } [Fact] @@ -145,7 +153,8 @@ public void BuildSearchQueryWithFilterReturnsValidQuery() s_jsonSerializerOptions, top: 3, searchOptions, - this._model); + this._model, + hasNamedVectors: true); // Assert Assert.Contains(ExpectedFirstSubquery, query); @@ -170,7 +179,8 @@ public void BuildSearchQueryWithInvalidFilterValueThrowsException() s_jsonSerializerOptions, top: 3, searchOptions, - this._model)); + this._model, + hasNamedVectors: true)); } [Fact] @@ -191,7 +201,8 @@ public void BuildSearchQueryWithNonExistentPropertyInFilterThrowsException() s_jsonSerializerOptions, top: 3, searchOptions, - this._model)); + this._model, + hasNamedVectors: true)); } #region private diff --git a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs index 78763fc0a59a..8524c03ae574 100644 --- a/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs +++ b/dotnet/src/Connectors/Connectors.Weaviate.UnitTests/WeaviateVectorStoreRecordMapperTests.cs @@ -28,27 +28,10 @@ public sealed class WeaviateVectorStoreRecordMapperTests } }; - private readonly WeaviateVectorStoreRecordMapper _sut = - new( - "CollectionName", - new WeaviateModelBuilder() - .Build( - typeof(Dictionary), - new VectorStoreRecordDefinition - { - Properties = - [ - new VectorStoreRecordKeyProperty("HotelId", typeof(Guid)), - new VectorStoreRecordDataProperty("HotelName", typeof(string)), - new VectorStoreRecordDataProperty("Tags", typeof(List)), - new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory), 10) - ] - }, - s_jsonSerializerOptions), - s_jsonSerializerOptions); - - [Fact] - public void MapFromDataToStorageModelReturnsValidObject() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromDataToStorageModelReturnsValidObject(bool hasNamedVectors) { // Arrange var hotel = new WeaviateHotel @@ -59,8 +42,10 @@ public void MapFromDataToStorageModelReturnsValidObject() DescriptionEmbedding = new ReadOnlyMemory([1f, 2f, 3f]) }; + var sut = GetMapper(hasNamedVectors); + // Act - var document = this._sut.MapFromDataToStorageModel(hotel); + var document = sut.MapFromDataToStorageModel(hotel); // Assert Assert.NotNull(document); @@ -68,11 +53,16 @@ public void MapFromDataToStorageModelReturnsValidObject() Assert.Equal("55555555-5555-5555-5555-555555555555", document["id"]!.GetValue()); Assert.Equal("Test Name", document["properties"]!["hotelName"]!.GetValue()); Assert.Equal(["tag1", "tag2"], document["properties"]!["tags"]!.AsArray().Select(l => l!.GetValue())); - Assert.Equal([1f, 2f, 3f], document["vectors"]!["descriptionEmbedding"]!.AsArray().Select(l => l!.GetValue())); + + var vectorNode = hasNamedVectors ? document["vectors"]!["descriptionEmbedding"] : document["vector"]; + + Assert.Equal([1f, 2f, 3f], vectorNode!.AsArray().Select(l => l!.GetValue())); } - [Fact] - public void MapFromStorageToDataModelReturnsValidObject() + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MapFromStorageToDataModelReturnsValidObject(bool hasNamedVectors) { // Arrange var document = new JsonObject @@ -84,10 +74,22 @@ public void MapFromStorageToDataModelReturnsValidObject() document["properties"]!["hotelName"] = "Test Name"; document["properties"]!["tags"] = new JsonArray(new List { "tag1", "tag2" }.Select(l => JsonValue.Create(l)).ToArray()); - document["vectors"]!["descriptionEmbedding"] = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); + + var vectorNode = new JsonArray(new List { 1f, 2f, 3f }.Select(l => JsonValue.Create(l)).ToArray()); + + if (hasNamedVectors) + { + document["vectors"]!["descriptionEmbedding"] = vectorNode; + } + else + { + document["vector"] = vectorNode; + } + + var sut = GetMapper(hasNamedVectors); // Act - var hotel = this._sut.MapFromStorageToDataModel(document, new() { IncludeVectors = true }); + var hotel = sut.MapFromStorageToDataModel(document, new() { IncludeVectors = true }); // Assert Assert.NotNull(hotel); @@ -97,4 +99,27 @@ public void MapFromStorageToDataModelReturnsValidObject() Assert.Equal(["tag1", "tag2"], hotel.Tags); Assert.True(new ReadOnlyMemory([1f, 2f, 3f]).Span.SequenceEqual(hotel.DescriptionEmbedding!.Value.Span)); } + + #region private + + private static WeaviateVectorStoreRecordMapper GetMapper(bool hasNamedVectors) => new( + "CollectionName", + hasNamedVectors, + new WeaviateModelBuilder(hasNamedVectors) + .Build( + typeof(Dictionary), + new VectorStoreRecordDefinition + { + Properties = + [ + new VectorStoreRecordKeyProperty("HotelId", typeof(Guid)), + new VectorStoreRecordDataProperty("HotelName", typeof(string)), + new VectorStoreRecordDataProperty("Tags", typeof(List)), + new VectorStoreRecordVectorProperty("DescriptionEmbedding", typeof(ReadOnlyMemory), 10) + ] + }, + s_jsonSerializerOptions), + s_jsonSerializerOptions); + + #endregion } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs index 9d4b065d6c4c..4222608ab7c0 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateBatchConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.CRUD; -public class WeaviateBatchConformanceTests(WeaviateSimpleModelFixture fixture) - : BatchConformanceTests(fixture), IClassFixture +public class WeaviateBatchConformanceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : BatchConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateBatchConformanceTests_UnnamedVector(WeaviateSimpleModelUnnamedVectorFixture fixture) + : BatchConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs index 7e024a7a50bb..62825fac4ab1 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateDynamicRecordConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.CRUD; -public class WeaviateDynamicRecordConformanceTests(WeaviateDynamicDataModelFixture fixture) - : DynamicDataModelConformanceTests(fixture), IClassFixture +public class WeaviateDynamicRecordConformanceTests_NamedVectors(WeaviateDynamicDataModelNamedVectorsFixture fixture) + : DynamicDataModelConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateDynamicRecordConformanceTests_UnnamedVector(WeaviateDynamicDataModelUnnamedVectorFixture fixture) + : DynamicDataModelConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs index 016ab64870e6..20d5888ad851 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateNoVectorConformanceTests.cs @@ -12,7 +12,7 @@ public class WeaviateNoVectorConformanceTests(WeaviateNoVectorConformanceTests.F { public new class Fixture : NoVectorConformanceTests.Fixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; /// /// Weaviate collections must start with an uppercase letter. diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs index 3beb7b6e70e5..c2ad732eb59d 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/CRUD/WeaviateRecordConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.CRUD; -public class WeaviateRecordConformanceTests(WeaviateSimpleModelFixture fixture) - : RecordConformanceTests(fixture), IClassFixture +public class WeaviateRecordConformanceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : RecordConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateRecordConformanceTests_UnnamedVector(WeaviateSimpleModelUnnamedVectorFixture fixture) + : RecordConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs index e839b02ad942..3c817890ac16 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Collections/WeaviateCollectionConformanceTests.cs @@ -6,7 +6,12 @@ namespace WeaviateIntegrationTests.Collections; -public class WeaviateCollectionConformanceTests(WeaviateFixture fixture) - : CollectionConformanceTests(fixture), IClassFixture +public class WeaviateCollectionConformanceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture +{ +} + +public class WeaviateCollectionConformanceTests_UnnamedVector(WeaviateSimpleModelUnnamedVectorFixture fixture) + : CollectionConformanceTests(fixture), IClassFixture { } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs index 32082232591d..f8f76dd27943 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicFilterTests.cs @@ -65,6 +65,8 @@ public override Task Equal_with_string_is_not_Contains() public new class Fixture : BasicFilterTests.Fixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; } } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs index 312ac21b8372..c03a8fd8076b 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Filter/WeaviateBasicQueryTests.cs @@ -65,7 +65,7 @@ public override Task Equal_with_string_is_not_Contains() public new class Fixture : BasicQueryTests.QueryFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs index 30d6bc0516f5..1b386273b6d6 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/HybridSearch/WeaviateKeywordVectorizedHybridSearchTests.cs @@ -7,16 +7,16 @@ namespace WeaviateIntegrationTests.HybridSearch; -public class WeaviateKeywordVectorizedHybridSearchTests( - WeaviateKeywordVectorizedHybridSearchTests.VectorAndStringFixture vectorAndStringFixture, - WeaviateKeywordVectorizedHybridSearchTests.MultiTextFixture multiTextFixture) +public class WeaviateKeywordVectorizedHybridSearchTests_NamedVectors( + WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.VectorAndStringFixture vectorAndStringFixture, + WeaviateKeywordVectorizedHybridSearchTests_NamedVectors.MultiTextFixture multiTextFixture) : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), - IClassFixture, - IClassFixture + IClassFixture, + IClassFixture { public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; @@ -25,7 +25,33 @@ public class WeaviateKeywordVectorizedHybridSearchTests( public new class MultiTextFixture : KeywordVectorizedHybridSearchComplianceTests.MultiTextFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + protected override string CollectionName => "MultiTextHybridSearch"; + } +} + +public class WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector( + WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.VectorAndStringFixture vectorAndStringFixture, + WeaviateKeywordVectorizedHybridSearchTests_UnnamedVector.MultiTextFixture multiTextFixture) + : KeywordVectorizedHybridSearchComplianceTests(vectorAndStringFixture, multiTextFixture), + IClassFixture, + IClassFixture +{ + public new class VectorAndStringFixture : KeywordVectorizedHybridSearchComplianceTests.VectorAndStringFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; + + protected override string CollectionName => "VectorAndStringHybridSearch"; + } + + public new class MultiTextFixture : KeywordVectorizedHybridSearchComplianceTests.MultiTextFixture + { + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs index 1745a902a348..831f05734d6b 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/TestContainer/WeaviateBuilder.cs @@ -8,7 +8,7 @@ namespace WeaviateIntegrationTests.Support.TestContainer; public sealed class WeaviateBuilder : ContainerBuilder { - public const string WeaviateImage = "semitechnologies/weaviate:1.26.4"; + public const string WeaviateImage = "semitechnologies/weaviate:1.28.12"; public const ushort WeaviateHttpPort = 8080; public const ushort WeaviateGrpcPort = 50051; diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs index cd69271c5a12..038de8fa5fd2 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateDynamicDataModelFixture.cs @@ -6,9 +6,21 @@ namespace WeaviateIntegrationTests.Support; public class WeaviateDynamicDataModelFixture : DynamicDataModelFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - protected override string CollectionName => $"A{Guid.NewGuid():N}"; + protected override string CollectionName => this.GetUniqueCollectionName(); + + public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; +} + +public class WeaviateDynamicDataModelNamedVectorsFixture : WeaviateDynamicDataModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; +} + +public class WeaviateDynamicDataModelUnnamedVectorFixture : WeaviateDynamicDataModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateFixture.cs deleted file mode 100644 index ac3b64f89006..000000000000 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateFixture.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using VectorDataSpecificationTests.Support; - -namespace WeaviateIntegrationTests.Support; - -public class WeaviateFixture : VectorStoreFixture -{ - public override TestStore TestStore => WeaviateTestStore.Instance; - - // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. - // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; -} diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs index 0fe7c713e46b..a10525cbb906 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateSimpleModelFixture.cs @@ -6,9 +6,23 @@ namespace WeaviateIntegrationTests.Support; public class WeaviateSimpleModelFixture : SimpleModelFixture { - public override TestStore TestStore => WeaviateTestStore.Instance; + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; + + protected override string DistanceFunction => Microsoft.Extensions.VectorData.DistanceFunction.CosineDistance; // Weaviate requires the name to start with a capital letter and not contain any chars other than a-Z and 0-9. // Source: https://weaviate.io/developers/weaviate/starter-guides/managing-collections#collection--property-names - protected override string CollectionName => $"A{Guid.NewGuid():N}"; + protected override string CollectionName => this.GetUniqueCollectionName(); + + public override string GetUniqueCollectionName() => $"A{Guid.NewGuid():N}"; +} + +public class WeaviateSimpleModelNamedVectorsFixture : WeaviateSimpleModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.NamedVectorsInstance; +} + +public class WeaviateSimpleModelUnnamedVectorFixture : WeaviateSimpleModelFixture +{ + public override TestStore TestStore => WeaviateTestStore.UnnamedVectorInstance; } diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs index a7700149dad3..76aa72077a3a 100644 --- a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/Support/WeaviateTestStore.cs @@ -12,9 +12,11 @@ namespace WeaviateIntegrationTests.Support; public sealed class WeaviateTestStore : TestStore { - public static WeaviateTestStore Instance { get; } = new(); + public static WeaviateTestStore NamedVectorsInstance { get; } = new(hasNamedVectors: true); + public static WeaviateTestStore UnnamedVectorInstance { get; } = new(hasNamedVectors: false); private readonly WeaviateContainer _container = new WeaviateBuilder().Build(); + private readonly bool _hasNamedVectors; public HttpClient? _httpClient { get; private set; } private WeaviateVectorStore? _defaultVectorStore; @@ -22,20 +24,13 @@ public sealed class WeaviateTestStore : TestStore public override IVectorStore DefaultVectorStore => this._defaultVectorStore ?? throw new InvalidOperationException("Not initialized"); - public override string DefaultDistanceFunction => DistanceFunction.CosineDistance; - - public WeaviateVectorStore GetVectorStore(WeaviateVectorStoreOptions options) - => new(this.Client, options); - - private WeaviateTestStore() - { - } + private WeaviateTestStore(bool hasNamedVectors) => this._hasNamedVectors = hasNamedVectors; protected override async Task StartAsync() { await this._container.StartAsync(); this._httpClient = new HttpClient { BaseAddress = new Uri($"http://localhost:{this._container.GetMappedPublicPort(WeaviateBuilder.WeaviateHttpPort)}/v1/") }; - this._defaultVectorStore = new(this._httpClient); + this._defaultVectorStore = new(this._httpClient, new() { HasNamedVectors = this._hasNamedVectors }); } protected override Task StopAsync() diff --git a/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs new file mode 100644 index 000000000000..0c6a5aadd390 --- /dev/null +++ b/dotnet/src/VectorDataIntegrationTests/WeaviateIntegrationTests/VectorSearch/WeaviateVectorSearchDistanceFunctionComplianceTests.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Microsoft.Extensions.VectorData; +using VectorDataSpecificationTests.VectorSearch; +using WeaviateIntegrationTests.Support; +using Xunit; + +namespace WeaviateIntegrationTests.VectorSearch; + +public class WeaviateVectorSearchDistanceFunctionComplianceTests_NamedVectors(WeaviateSimpleModelNamedVectorsFixture fixture) + : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + + /// + /// Tests vector search using , computing -(u · v) as a distance metric per Weaviate's convention. + /// Expects scores of -1 (exact match), 1 (opposite), and 0 (orthogonal), sorted ascending ([0, 2, 1]), with lower scores indicating closer matches. + /// . + /// + public override Task NegativeDotProductSimilarity() => this.SimpleSearch(DistanceFunction.NegativeDotProductSimilarity, -1, 1, 0, [0, 2, 1]); +} + +public class WeaviateVectorSearchDistanceFunctionComplianceTests_UnnamedVector(WeaviateDynamicDataModelNamedVectorsFixture fixture) + : VectorSearchDistanceFunctionComplianceTests(fixture), IClassFixture +{ + public override Task CosineSimilarity() => Assert.ThrowsAsync(base.CosineSimilarity); + + public override Task DotProductSimilarity() => Assert.ThrowsAsync(base.DotProductSimilarity); + + public override Task EuclideanDistance() => Assert.ThrowsAsync(base.EuclideanDistance); + + /// + /// Tests vector search using , computing -(u · v) as a distance metric per Weaviate's convention. + /// Expects scores of -1 (exact match), 1 (opposite), and 0 (orthogonal), sorted ascending ([0, 2, 1]), with lower scores indicating closer matches. + /// . + /// + public override Task NegativeDotProductSimilarity() => this.SimpleSearch(DistanceFunction.NegativeDotProductSimilarity, -1, 1, 0, [0, 2, 1]); +}