.Net: Add qdrant and weaviate keyword hybrid search implementations and tests (#10505)

### Motivation and Context

#9399

### Description

- Add Qdrant keyword hybrid search
- Add Weaviate keyword hybrid search
- Add integrations tests for both.

### Contribution Checklist

<!-- Before submitting this PR, please make sure: -->

- [x] The code builds clean without any errors or warnings
- [x] The PR follows the [SK Contribution
Guidelines](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md)
and the [pre-submission formatting
script](https://github.com/microsoft/semantic-kernel/blob/main/CONTRIBUTING.md#development-scripts)
raises no violations
- [x] All unit tests pass, and I have added new tests where possible
- [x] I didn't break anyone 😄

Author: westey-m

dotnet/src/Connectors/Connectors.Memory.Qdrant/QdrantVectorStoreRecordCollection.cs

@@ -18,7 +18,10 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant;
 /// </summary>
 /// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam>
 #pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public sealed class QdrantVectorStoreRecordCollection<TRecord> : IVectorStoreRecordCollection<ulong, TRecord>, IVectorStoreRecordCollection<Guid, TRecord>
+public sealed class QdrantVectorStoreRecordCollection<TRecord> :
+    IVectorStoreRecordCollection<ulong, TRecord>,
+    IVectorStoreRecordCollection<Guid, TRecord>,
+    IKeywordVectorizedHybridSearch<TRecord>
 #pragma warning restore CA1711 // Identifiers should not have incorrect suffix
 {
     /// <summary>A set of types that a key on the provided model may have.</summary>
@@ -31,6 +34,9 @@ public sealed class QdrantVectorStoreRecordCollection<TRecord> : IVectorStoreRec
     /// <summary>The default options for vector search.</summary>
     private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new();
 
+    /// <summary>The default options for hybrid vector search.</summary>
+    private static readonly KeywordVectorizedHybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new();
+
     /// <summary>The name of this database for telemetry purposes.</summary>
     private const string DatabaseName = "Qdrant";
 
@@ -459,19 +465,11 @@ private async IAsyncEnumerable<TRecord> GetBatchByPointIdAsync<TKey>(
     /// <inheritdoc />
     public async Task<VectorSearchResults<TRecord>> VectorizedSearchAsync<TVector>(TVector vector, VectorSearchOptions? options = null, CancellationToken cancellationToken = default)
     {
-        Verify.NotNull(vector);
-
-        if (this._propertyReader.FirstVectorPropertyName is null)
-        {
-            throw new InvalidOperationException("The collection does not have any vector fields, so vector search is not possible.");
-        }
-
-        if (vector is not ReadOnlyMemory<float> floatVector)
-        {
-            throw new NotSupportedException($"The provided vector type {vector.GetType().FullName} is not supported by the Qdrant connector.");
-        }
+        var floatVector = VerifyVectorParam(vector);
 
+        // Resolve options.
         var internalOptions = options ?? s_defaultVectorSearchOptions;
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrFirst(internalOptions.VectorPropertyName);
 
         // Build filter object.
         var filter = QdrantVectorStoreCollectionSearchMapping.BuildFilter(internalOptions.Filter, this._propertyReader.StoragePropertyNamesMap);
@@ -480,7 +478,7 @@ public async Task<VectorSearchResults<TRecord>> VectorizedSearchAsync<TVector>(T
         string? vectorName = null;
         if (this._options.HasNamedVectors)
         {
-            vectorName = this.ResolveVectorFieldName(internalOptions.VectorPropertyName);
+            vectorName = this._propertyReader.GetStoragePropertyName(vectorProperty.DataModelPropertyName);
         }
 
         // Specify whether to include vectors in the search results.
@@ -517,29 +515,87 @@ public async Task<VectorSearchResults<TRecord>> VectorizedSearchAsync<TVector>(T
         return new VectorSearchResults<TRecord>(mappedResults.ToAsyncEnumerable());
     }
 
-    /// <summary>
-    /// Resolve the vector field name to use for a search by using the storage name for the field name from options
-    /// if available, and falling back to the first vector field name if not.
-    /// </summary>
-    /// <param name="optionsVectorFieldName">The vector field name provided via options.</param>
-    /// <returns>The resolved vector field name.</returns>
-    /// <exception cref="InvalidOperationException">Thrown if the provided field name is not a valid field name.</exception>
-    private string ResolveVectorFieldName(string? optionsVectorFieldName)
+    /// <inheritdoc />
+    public async Task<VectorSearchResults<TRecord>> KeywordVectorizedHybridSearch<TVector>(TVector vector, ICollection<string> keywords, KeywordVectorizedHybridSearchOptions? options = null, CancellationToken cancellationToken = default)
     {
-        string? vectorFieldName;
-        if (!string.IsNullOrWhiteSpace(optionsVectorFieldName))
+        var floatVector = VerifyVectorParam(vector);
+
+        // Resolve options.
+        var internalOptions = options ?? s_defaultKeywordVectorizedHybridSearchOptions;
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrFirst(internalOptions.VectorPropertyName);
+
+        // Build filter object.
+        var filter = QdrantVectorStoreCollectionSearchMapping.BuildFilter(internalOptions.Filter, this._propertyReader.StoragePropertyNamesMap);
+        var textDataProperty = this._propertyReader.GetFullTextDataPropertyOrSingle(internalOptions.FullTextPropertyName);
+        var textDataPropertyName = this._propertyReader.GetStoragePropertyName(textDataProperty.DataModelPropertyName);
+
+        // Specify the vector name if named vectors are used.
+        string? vectorName = null;
+        if (this._options.HasNamedVectors)
         {
-            if (!this._propertyReader.StoragePropertyNamesMap.TryGetValue(optionsVectorFieldName!, out vectorFieldName))
+            vectorName = this._propertyReader.GetStoragePropertyName(vectorProperty.DataModelPropertyName);
+        }
+
+        // Specify whether to include vectors in the search results.
+        var vectorsSelector = new WithVectorsSelector();
+        vectorsSelector.Enable = internalOptions.IncludeVectors;
+
+        // Build the vector query.
+        var vectorQuery = new PrefetchQuery
+        {
+            Filter = filter,
+            Query = new Query
             {
-                throw new InvalidOperationException($"The collection does not have a vector field named '{optionsVectorFieldName}'.");
-            }
+                Nearest = new VectorInput(floatVector.ToArray()),
+            },
+        };
+
+        if (this._options.HasNamedVectors)
+        {
+            vectorQuery.Using = vectorName;
         }
-        else
+
+        // Build the keyword query.
+        var keywordFilter = filter.Clone();
+        var keywordSubFilter = new Filter();
+        foreach (string keyword in keywords)
         {
-            vectorFieldName = this._propertyReader.FirstVectorPropertyStoragePropertyName;
+            keywordSubFilter.Should.Add(new Condition() { Field = new FieldCondition() { Key = textDataPropertyName, Match = new Match { Text = keyword } } });
         }
+        keywordFilter.Must.Add(new Condition() { Filter = keywordSubFilter });
+        var keywordQuery = new PrefetchQuery
+        {
+            Filter = keywordFilter,
+        };
+
+        // Build the fusion query.
+        var fusionQuery = new Query
+        {
+            Fusion = Fusion.Rrf,
+        };
+
+        // Execute Search.
+        var points = await this.RunOperationAsync(
+            "Query",
+            () => this._qdrantClient.QueryAsync(
+                this.CollectionName,
+                prefetch: new List<PrefetchQuery>() { vectorQuery, keywordQuery },
+                query: fusionQuery,
+                limit: (ulong)internalOptions.Top,
+                offset: (ulong)internalOptions.Skip,
+                vectorsSelector: vectorsSelector,
+                cancellationToken: cancellationToken)).ConfigureAwait(false);
 
-        return vectorFieldName!;
+        // Map to data model.
+        var mappedResults = points.Select(point => QdrantVectorStoreCollectionSearchMapping.MapScoredPointToVectorSearchResult(
+                point,
+                this._mapper,
+                internalOptions.IncludeVectors,
+                DatabaseName,
+                this._collectionName,
+                "Query"));
+
+        return new VectorSearchResults<TRecord>(mappedResults.ToAsyncEnumerable());
     }
 
     /// <summary>
@@ -588,4 +644,16 @@ private async Task<T> RunOperationAsync<T>(string operationName, Func<Task<T>> o
             };
         }
     }
+
+    private static ReadOnlyMemory<float> VerifyVectorParam<TVector>(TVector vector)
+    {
+        Verify.NotNull(vector);
+
+        if (vector is not ReadOnlyMemory<float> floatVector)
+        {
+            throw new NotSupportedException($"The provided vector type {vector.GetType().FullName} is not supported by the Qdrant connector.");
+        }
+
+        return floatVector;
+    }
 }

dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateConstants.cs

@@ -19,6 +19,9 @@ internal sealed class WeaviateConstants
     /// <summary>Score property name in Weaviate.</summary>
     internal const string ScorePropertyName = "distance";
 
+    /// <summary>Score property name for hybrid search in Weaviate.</summary>
+    internal const string HybridScorePropertyName = "score";
+
     /// <summary>Additional properties property name in Weaviate.</summary>
     internal const string AdditionalPropertiesPropertyName = "_additional";
 }

dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreCollectionSearchMapping.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System.Text.Json;
 using System.Text.Json.Nodes;
 
 namespace Microsoft.SemanticKernel.Connectors.Weaviate;
@@ -12,11 +13,17 @@ internal static class WeaviateVectorStoreCollectionSearchMapping
     /// <summary>
     /// Maps vector search result to the format, which is processable by <see cref="WeaviateVectorStoreRecordMapper{TRecord}"/>.
     /// </summary>
-    public static (JsonObject StorageModel, double? Score) MapSearchResult(JsonNode result)
+    public static (JsonObject StorageModel, double? Score) MapSearchResult(JsonNode result, string scorePropertyName)
     {
         var additionalProperties = result[WeaviateConstants.AdditionalPropertiesPropertyName];
 
-        var score = additionalProperties?[WeaviateConstants.ScorePropertyName]?.GetValue<double>();
+        var scoreProperty = additionalProperties?[scorePropertyName];
+        double? score = scoreProperty?.GetValueKind() switch
+        {
+            JsonValueKind.Number => scoreProperty.GetValue<double>(),
+            JsonValueKind.String => double.Parse(scoreProperty.GetValue<string>()),
+            _ => null
+        };
 
         var id = additionalProperties?[WeaviateConstants.ReservedKeyPropertyName];
         var vectors = additionalProperties?[WeaviateConstants.ReservedVectorPropertyName];

dotnet/src/Connectors/Connectors.Memory.Weaviate/WeaviateVectorStoreRecordCollection.cs

@@ -22,7 +22,7 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate;
 /// </summary>
 /// <typeparam name="TRecord">The data model to use for adding, updating and retrieving data from storage.</typeparam>
 #pragma warning disable CA1711 // Identifiers should not have incorrect suffix
-public sealed class WeaviateVectorStoreRecordCollection<TRecord> : IVectorStoreRecordCollection<Guid, TRecord>
+public sealed class WeaviateVectorStoreRecordCollection<TRecord> : IVectorStoreRecordCollection<Guid, TRecord>, IKeywordVectorizedHybridSearch<TRecord>
 #pragma warning restore CA1711 // Identifiers should not have incorrect suffix
 {
     /// <summary>The name of this database for telemetry purposes.</summary>
@@ -86,6 +86,9 @@ public sealed class WeaviateVectorStoreRecordCollection<TRecord> : IVectorStoreR
     /// <summary>The default options for vector search.</summary>
     private static readonly VectorSearchOptions s_defaultVectorSearchOptions = new();
 
+    /// <summary>The default options for hybrid vector search.</summary>
+    private static readonly KeywordVectorizedHybridSearchOptions s_defaultKeywordVectorizedHybridSearchOptions = new();
+
     /// <summary><see cref="HttpClient"/> that is used to interact with Weaviate API.</summary>
     private readonly HttpClient _httpClient;
 
@@ -343,39 +346,63 @@ public async Task<VectorSearchResults<TRecord>> VectorizedSearchAsync<TVector>(
     {
         const string OperationName = "VectorSearch";
 
-        Verify.NotNull(vector);
+        VerifyVectorParam(vector);
 
-        var vectorType = vector.GetType();
+        var searchOptions = options ?? s_defaultVectorSearchOptions;
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrFirst(searchOptions.VectorPropertyName);
 
-        if (!s_supportedVectorTypes.Contains(vectorType))
-        {
-            throw new NotSupportedException(
-                $"The provided vector type {vectorType.FullName} is not supported by the Azure CosmosDB NoSQL connector. " +
-                $"Supported types are: {string.Join(", ", s_supportedVectorTypes.Select(l => l.FullName))}");
-        }
+        var vectorPropertyName = this._propertyReader.GetJsonPropertyName(vectorProperty.DataModelPropertyName);
+        var fields = this._propertyReader.DataPropertyJsonNames;
 
-        var searchOptions = options ?? s_defaultVectorSearchOptions;
-        var vectorProperty = this.GetVectorPropertyForSearch(searchOptions.VectorPropertyName);
+        var query = WeaviateVectorStoreRecordCollectionQueryBuilder.BuildSearchQuery(
+            vector,
+            this.CollectionName,
+            vectorPropertyName,
+            this._propertyReader.KeyPropertyName,
+            s_jsonSerializerOptions,
+            searchOptions,
+            this._propertyReader.JsonPropertyNamesMap,
+            this._propertyReader.VectorPropertyJsonNames,
+            this._propertyReader.DataPropertyJsonNames);
 
-        if (vectorProperty is null)
-        {
-            throw new InvalidOperationException("The collection does not have any vector properties, so vector search is not possible.");
-        }
+        return await this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.ScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false);
+    }
+
+    /// <inheritdoc />
+    public async Task<VectorSearchResults<TRecord>> KeywordVectorizedHybridSearch<TVector>(TVector vector, ICollection<string> keywords, KeywordVectorizedHybridSearchOptions? options = null, CancellationToken cancellationToken = default)
+    {
+        const string OperationName = "HybridSearch";
+
+        VerifyVectorParam(vector);
+
+        var searchOptions = options ?? s_defaultKeywordVectorizedHybridSearchOptions;
+        var vectorProperty = this._propertyReader.GetVectorPropertyOrFirst(searchOptions.VectorPropertyName);
+        var textDataProperty = this._propertyReader.GetFullTextDataPropertyOrSingle(searchOptions.FullTextPropertyName);
 
         var vectorPropertyName = this._propertyReader.GetJsonPropertyName(vectorProperty.DataModelPropertyName);
+        var textDataPropertyName = this._propertyReader.GetJsonPropertyName(textDataProperty.DataModelPropertyName);
         var fields = this._propertyReader.DataPropertyJsonNames;
 
-        var query = WeaviateVectorStoreRecordCollectionQueryBuilder.BuildSearchQuery(
+        var query = WeaviateVectorStoreRecordCollectionQueryBuilder.BuildHybridSearchQuery(
             vector,
+            string.Join(" ", keywords),
             this.CollectionName,
             vectorPropertyName,
             this._propertyReader.KeyPropertyName,
+            textDataPropertyName,
             s_jsonSerializerOptions,
             searchOptions,
             this._propertyReader.JsonPropertyNamesMap,
             this._propertyReader.VectorPropertyJsonNames,
             this._propertyReader.DataPropertyJsonNames);
 
+        return await this.ExecuteQueryAsync(query, searchOptions.IncludeVectors, WeaviateConstants.HybridScorePropertyName, OperationName, cancellationToken).ConfigureAwait(false);
+    }
+
+    #region private
+
+    private async Task<VectorSearchResults<TRecord>> ExecuteQueryAsync(string query, bool includeVectors, string scorePropertyName, string operationName, CancellationToken cancellationToken)
+    {
         using var request = new WeaviateVectorSearchRequest(query).Build();
 
         var (responseModel, content) = await this.ExecuteRequestWithResponseContentAsync<WeaviateVectorSearchResponse>(request, cancellationToken).ConfigureAwait(false);
@@ -388,28 +415,26 @@ public async Task<VectorSearchResults<TRecord>> VectorizedSearchAsync<TVector>(
             {
                 VectorStoreType = DatabaseName,
                 CollectionName = this.CollectionName,
-                OperationName = OperationName
+                OperationName = operationName
             };
         }
 
         var mappedResults = collectionResults.Where(x => x is not null).Select(result =>
         {
-            var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!);
+            var (storageModel, score) = WeaviateVectorStoreCollectionSearchMapping.MapSearchResult(result!, scorePropertyName);
 
             var record = VectorStoreErrorHandler.RunModelConversion(
                 DatabaseName,
                 this.CollectionName,
-                OperationName,
-                () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = searchOptions.IncludeVectors }));
+                operationName,
+                () => this._mapper.MapFromStorageToDataModel(storageModel, new() { IncludeVectors = includeVectors }));
 
             return new VectorSearchResult<TRecord>(record, score);
         });
 
         return new VectorSearchResults<TRecord>(mappedResults.ToAsyncEnumerable());
     }
 
-    #region private
-
     private Task<HttpResponseMessage> ExecuteRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken)
     {
         request.RequestUri = new Uri(this._endpoint, request.RequestUri!);
@@ -469,33 +494,6 @@ private async Task<T> RunOperationAsync<T>(string operationName, Func<Task<T>> o
         }
     }
 
-    /// <summary>
-    /// Get vector property to use for a search by using the storage name for the field name from options
-    /// if available, and falling back to the first vector property in <typeparamref name="TRecord"/> if not.
-    /// </summary>
-    /// <param name="vectorFieldName">The vector field name.</param>
-    /// <exception cref="InvalidOperationException">Thrown if the provided field name is not a valid field name.</exception>
-    private VectorStoreRecordVectorProperty? GetVectorPropertyForSearch(string? vectorFieldName)
-    {
-        // If vector property name is provided in options, try to find it in schema or throw an exception.
-        if (!string.IsNullOrWhiteSpace(vectorFieldName))
-        {
-            // Check vector properties by data model property name.
-            var vectorProperty = this._propertyReader.VectorProperties
-                .FirstOrDefault(l => l.DataModelPropertyName.Equals(vectorFieldName, StringComparison.Ordinal));
-
-            if (vectorProperty is not null)
-            {
-                return vectorProperty;
-            }
-
-            throw new InvalidOperationException($"The {typeof(TRecord).FullName} type does not have a vector property named '{vectorFieldName}'.");
-        }
-
-        // If vector property is not provided in options, return first vector property from schema.
-        return this._propertyReader.VectorProperty;
-    }
-
     /// <summary>
     /// Returns custom mapper, generic data model mapper or default record mapper.
     /// </summary>
@@ -528,5 +526,19 @@ private IVectorStoreRecordMapper<TRecord, JsonObject> InitializeMapper()
             s_jsonSerializerOptions);
     }
 
+    private static void VerifyVectorParam<TVector>(TVector vector)
+    {
+        Verify.NotNull(vector);
+
+        var vectorType = vector.GetType();
+
+        if (!s_supportedVectorTypes.Contains(vectorType))
+        {
+            throw new NotSupportedException(
+                $"The provided vector type {vectorType.FullName} is not supported by the Weaviate connector. " +
+                $"Supported types are: {string.Join(", ", s_supportedVectorTypes.Select(l => l.FullName))}");
+        }
+    }
+
     #endregion
 }