-
Notifications
You must be signed in to change notification settings - Fork 4k
.Net: IVectorStore implementation for Azure SQL #10623
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
adamsitnik
merged 36 commits into
microsoft:feature-vector-data-preb1
from
adamsitnik:sqlServerNew
Mar 7, 2025
Merged
Changes from all commits
Commits
Show all changes
36 commits
Select commit
Hold shift + click to select a range
160e08e
add new test project, move the existing tests to it
adamsitnik 186fda2
port existing tests to Testcontainers.MsSql and re-enable them
adamsitnik 22495b9
Revert "port existing tests to Testcontainers.MsSql and re-enable them"
adamsitnik 3c3fd4a
Merge remote-tracking branch 'upstream/feature-vector-data-preb1' int…
adamsitnik 179f56e
implement the tests using the new pattern, provide implementation tha…
adamsitnik 486d028
implement collection removal, existence check and creation
adamsitnik a41cac4
implement record insert and update (upsert)
adamsitnik 6dfb04a
implement delete operations
adamsitnik e905409
GetAsync and GetBatchAsync
adamsitnik 7c212da
refactor
adamsitnik 8d71b11
implement UpsertBatchAsync
adamsitnik 7f18352
implement SelectTableNames, read the code again and add TODOs for thi…
adamsitnik 32605da
ensure that parameter names are always valid
adamsitnik b4a73ee
add some comments
adamsitnik e8584be
support storing more types, support auto-generated keys
adamsitnik f397f3f
simplify: don't use a dedicated query for inserting a single record
adamsitnik ffc4b14
Merge remote-tracking branch 'upstream/feature-vector-data-preb1' int…
adamsitnik 7c8d2dc
vector search
adamsitnik 9e5ef1c
implement filtering by reusing a lot of code implemented by @roji
adamsitnik 080811f
reduce code duplication
adamsitnik c17021e
skip some tests, some polishing
adamsitnik 4669e91
remove a comment added by Copilot
adamsitnik ba0486f
Update dotnet/src/Connectors/VectorData.Abstractions/RecordAttributes…
adamsitnik 5bdaa8e
address code review feedback:
adamsitnik 1902c0b
address remaining feedback:
adamsitnik 3081305
implement IndexKind support for SqlServer and fix it for PostgreSQL:
adamsitnik 5b843aa
fix the build
adamsitnik 8bb8aea
throw for null inputs, do nothing for empty ones
adamsitnik f76b573
address code review feedback:
adamsitnik c40f341
Update dotnet/src/Connectors/VectorData.Abstractions/RecordDefinition…
adamsitnik 2fe49c0
Apply suggestions from code review
adamsitnik 4639a17
Merge remote-tracking branch 'upstream/feature-vector-data-preb1' int…
adamsitnik 0bdca76
address code review feedback:
adamsitnik 88419c3
Merge remote-tracking branch 'upstream/feature-vector-data-preb1' int…
adamsitnik 9ed18ab
remove AutoGenerate
adamsitnik c42c6cb
Apply suggestions from code review
adamsitnik File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
326 changes: 326 additions & 0 deletions
326
dotnet/src/Connectors/Connectors.Memory.Common/SqlFilterTranslator.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,326 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Diagnostics; | ||
| using System.Diagnostics.CodeAnalysis; | ||
| using System.Linq; | ||
| using System.Linq.Expressions; | ||
| using System.Reflection; | ||
| using System.Runtime.CompilerServices; | ||
| using System.Text; | ||
|
|
||
| namespace Microsoft.SemanticKernel.Connectors; | ||
|
|
||
| internal abstract class SqlFilterTranslator | ||
| { | ||
| private readonly IReadOnlyDictionary<string, string> _storagePropertyNames; | ||
| private readonly LambdaExpression _lambdaExpression; | ||
| private readonly ParameterExpression _recordParameter; | ||
| protected readonly StringBuilder _sql; | ||
|
|
||
| internal SqlFilterTranslator( | ||
| IReadOnlyDictionary<string, string> storagePropertyNames, | ||
| LambdaExpression lambdaExpression, | ||
| StringBuilder? sql = null) | ||
| { | ||
| this._storagePropertyNames = storagePropertyNames; | ||
| this._lambdaExpression = lambdaExpression; | ||
| Debug.Assert(lambdaExpression.Parameters.Count == 1); | ||
| this._recordParameter = lambdaExpression.Parameters[0]; | ||
| this._sql = sql ?? new(); | ||
| } | ||
|
|
||
| internal StringBuilder Clause => this._sql; | ||
|
|
||
| internal void Translate(bool appendWhere) | ||
| { | ||
| if (appendWhere) | ||
| { | ||
| this._sql.Append("WHERE "); | ||
| } | ||
|
|
||
| this.Translate(this._lambdaExpression.Body, null); | ||
| } | ||
|
|
||
| protected void Translate(Expression? node, Expression? parent) | ||
| { | ||
| switch (node) | ||
| { | ||
| case BinaryExpression binary: | ||
| this.TranslateBinary(binary); | ||
| return; | ||
|
|
||
| case ConstantExpression constant: | ||
| this.TranslateConstant(constant.Value); | ||
| return; | ||
|
|
||
| case MemberExpression member: | ||
| this.TranslateMember(member, parent); | ||
| return; | ||
|
|
||
| case MethodCallExpression methodCall: | ||
| this.TranslateMethodCall(methodCall); | ||
| return; | ||
|
|
||
| case UnaryExpression unary: | ||
| this.TranslateUnary(unary); | ||
| return; | ||
|
|
||
| default: | ||
| throw new NotSupportedException("Unsupported NodeType in filter: " + node?.NodeType); | ||
| } | ||
| } | ||
|
|
||
| protected void TranslateBinary(BinaryExpression binary) | ||
| { | ||
| // Special handling for null comparisons | ||
| switch (binary.NodeType) | ||
| { | ||
| case ExpressionType.Equal when IsNull(binary.Right): | ||
| this._sql.Append('('); | ||
| this.Translate(binary.Left, binary); | ||
| this._sql.Append(" IS NULL)"); | ||
| return; | ||
| case ExpressionType.NotEqual when IsNull(binary.Right): | ||
| this._sql.Append('('); | ||
| this.Translate(binary.Left, binary); | ||
| this._sql.Append(" IS NOT NULL)"); | ||
| return; | ||
|
|
||
| case ExpressionType.Equal when IsNull(binary.Left): | ||
| this._sql.Append('('); | ||
| this.Translate(binary.Right, binary); | ||
| this._sql.Append(" IS NULL)"); | ||
| return; | ||
| case ExpressionType.NotEqual when IsNull(binary.Left): | ||
| this._sql.Append('('); | ||
| this.Translate(binary.Right, binary); | ||
| this._sql.Append(" IS NOT NULL)"); | ||
| return; | ||
| } | ||
|
|
||
| this._sql.Append('('); | ||
| this.Translate(binary.Left, binary); | ||
|
|
||
| this._sql.Append(binary.NodeType switch | ||
| { | ||
| ExpressionType.Equal => " = ", | ||
| ExpressionType.NotEqual => " <> ", | ||
|
|
||
| ExpressionType.GreaterThan => " > ", | ||
| ExpressionType.GreaterThanOrEqual => " >= ", | ||
| ExpressionType.LessThan => " < ", | ||
| ExpressionType.LessThanOrEqual => " <= ", | ||
|
|
||
| ExpressionType.AndAlso => " AND ", | ||
| ExpressionType.OrElse => " OR ", | ||
|
|
||
| _ => throw new NotSupportedException("Unsupported binary expression node type: " + binary.NodeType) | ||
| }); | ||
|
|
||
| this.Translate(binary.Right, binary); | ||
| this._sql.Append(')'); | ||
|
|
||
| static bool IsNull(Expression expression) | ||
| => expression is ConstantExpression { Value: null } | ||
| || (TryGetCapturedValue(expression, out _, out var capturedValue) && capturedValue is null); | ||
| } | ||
|
|
||
| protected virtual void TranslateConstant(object? value) | ||
| { | ||
| // TODO: Nullable | ||
| switch (value) | ||
| { | ||
| case byte b: | ||
| this._sql.Append(b); | ||
| return; | ||
| case short s: | ||
| this._sql.Append(s); | ||
| return; | ||
| case int i: | ||
| this._sql.Append(i); | ||
| return; | ||
| case long l: | ||
| this._sql.Append(l); | ||
| return; | ||
|
|
||
| case string s: | ||
| this._sql.Append('\'').Append(s.Replace("'", "''")).Append('\''); | ||
| return; | ||
| case bool b: | ||
| this._sql.Append(b ? "TRUE" : "FALSE"); | ||
| return; | ||
| case Guid g: | ||
| this._sql.Append('\'').Append(g.ToString()).Append('\''); | ||
| return; | ||
|
|
||
| case DateTime dateTime: | ||
| case DateTimeOffset dateTimeOffset: | ||
| case Array: | ||
| throw new NotImplementedException(); | ||
|
|
||
| case null: | ||
| this._sql.Append("NULL"); | ||
| return; | ||
|
|
||
| default: | ||
| throw new NotSupportedException("Unsupported constant type: " + value.GetType().Name); | ||
| } | ||
| } | ||
|
|
||
| private void TranslateMember(MemberExpression memberExpression, Expression? parent) | ||
| { | ||
| switch (memberExpression) | ||
| { | ||
| case var _ when this.TryGetColumn(memberExpression, out var column): | ||
| this.TranslateColumn(column, memberExpression, parent); | ||
| return; | ||
|
|
||
| case var _ when TryGetCapturedValue(memberExpression, out var name, out var value): | ||
| this.TranslateCapturedVariable(name, value); | ||
| return; | ||
|
|
||
| default: | ||
| throw new NotSupportedException($"Member access for '{memberExpression.Member.Name}' is unsupported - only member access over the filter parameter are supported"); | ||
| } | ||
| } | ||
|
|
||
| protected virtual void TranslateColumn(string column, MemberExpression memberExpression, Expression? parent) | ||
| => this._sql.Append('"').Append(column).Append('"'); | ||
|
|
||
| protected abstract void TranslateCapturedVariable(string name, object? capturedValue); | ||
|
|
||
| private void TranslateMethodCall(MethodCallExpression methodCall) | ||
| { | ||
| switch (methodCall) | ||
| { | ||
| // Enumerable.Contains() | ||
| case { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains | ||
| when contains.Method.DeclaringType == typeof(Enumerable): | ||
| this.TranslateContains(source, item, methodCall); | ||
| return; | ||
|
|
||
| // List.Contains() | ||
| case | ||
| { | ||
| Method: | ||
| { | ||
| Name: nameof(Enumerable.Contains), | ||
| DeclaringType: { IsGenericType: true } declaringType | ||
| }, | ||
| Object: Expression source, | ||
| Arguments: [var item] | ||
| } when declaringType.GetGenericTypeDefinition() == typeof(List<>): | ||
| this.TranslateContains(source, item, methodCall); | ||
| return; | ||
|
|
||
| default: | ||
| throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}"); | ||
| } | ||
| } | ||
|
|
||
| private void TranslateContains(Expression source, Expression item, MethodCallExpression parent) | ||
| { | ||
| switch (source) | ||
| { | ||
| // Contains over array column (r => r.Strings.Contains("foo")) | ||
| case var _ when this.TryGetColumn(source, out _): | ||
| this.TranslateContainsOverArrayColumn(source, item, parent); | ||
| return; | ||
|
|
||
| // Contains over inline array (r => new[] { "foo", "bar" }.Contains(r.String)) | ||
| case NewArrayExpression newArray: | ||
| this.Translate(item, parent); | ||
| this._sql.Append(" IN ("); | ||
|
|
||
| var isFirst = true; | ||
| foreach (var element in newArray.Expressions) | ||
| { | ||
| if (isFirst) | ||
| { | ||
| isFirst = false; | ||
| } | ||
| else | ||
| { | ||
| this._sql.Append(", "); | ||
| } | ||
|
|
||
| this.Translate(element, parent); | ||
| } | ||
|
|
||
| this._sql.Append(')'); | ||
| return; | ||
|
|
||
| // Contains over captured array (r => arrayLocalVariable.Contains(r.String)) | ||
| case var _ when TryGetCapturedValue(source, out _, out var value): | ||
| this.TranslateContainsOverCapturedArray(source, item, parent, value); | ||
| return; | ||
|
|
||
| default: | ||
| throw new NotSupportedException("Unsupported Contains expression"); | ||
| } | ||
| } | ||
|
|
||
| protected abstract void TranslateContainsOverArrayColumn(Expression source, Expression item, MethodCallExpression parent); | ||
|
|
||
| protected abstract void TranslateContainsOverCapturedArray(Expression source, Expression item, MethodCallExpression parent, object? value); | ||
|
|
||
| private void TranslateUnary(UnaryExpression unary) | ||
| { | ||
| switch (unary.NodeType) | ||
| { | ||
| case ExpressionType.Not: | ||
| // Special handling for !(a == b) and !(a != b) | ||
| if (unary.Operand is BinaryExpression { NodeType: ExpressionType.Equal or ExpressionType.NotEqual } binary) | ||
| { | ||
| this.TranslateBinary( | ||
| Expression.MakeBinary( | ||
| binary.NodeType is ExpressionType.Equal ? ExpressionType.NotEqual : ExpressionType.Equal, | ||
| binary.Left, | ||
| binary.Right)); | ||
| return; | ||
| } | ||
|
|
||
| this._sql.Append("(NOT "); | ||
| this.Translate(unary.Operand, unary); | ||
| this._sql.Append(')'); | ||
| return; | ||
|
|
||
| default: | ||
| throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); | ||
| } | ||
| } | ||
|
|
||
| private bool TryGetColumn(Expression expression, [NotNullWhen(true)] out string? column) | ||
| { | ||
| if (expression is MemberExpression member && member.Expression == this._recordParameter) | ||
| { | ||
| if (!this._storagePropertyNames.TryGetValue(member.Member.Name, out column)) | ||
| { | ||
| throw new InvalidOperationException($"Property name '{member.Member.Name}' provided as part of the filter clause is not a valid property name."); | ||
| } | ||
|
|
||
| return true; | ||
| } | ||
|
|
||
| column = null; | ||
| return false; | ||
| } | ||
|
|
||
| private static bool TryGetCapturedValue(Expression expression, [NotNullWhen(true)] out string? name, out object? value) | ||
| { | ||
| if (expression is MemberExpression { Expression: ConstantExpression constant, Member: FieldInfo fieldInfo } | ||
| && constant.Type.Attributes.HasFlag(TypeAttributes.NestedPrivate) | ||
| && Attribute.IsDefined(constant.Type, typeof(CompilerGeneratedAttribute), inherit: true)) | ||
| { | ||
| name = fieldInfo.Name; | ||
| value = fieldInfo.GetValue(constant.Value); | ||
| return true; | ||
| } | ||
|
|
||
| name = null; | ||
| value = null; | ||
| return false; | ||
| } | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Since the representation of bool varies considerably across databases, this probably shouldn't be here (only in the overrides of this method in connector subtypes).