-
Notifications
You must be signed in to change notification settings - Fork 4k
.Net: Add Updated Memory Connector ADR and IMemoryRecordService interface with related classes. #6364
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
westey-m
merged 19 commits into
microsoft:feature-memory-service
from
westey-m:generic-vector-store-adr
Jun 5, 2024
Merged
.Net: Add Updated Memory Connector ADR and IMemoryRecordService interface with related classes. #6364
Changes from 15 commits
Commits
Show all changes
19 commits
Select commit
Hold shift + click to select a range
75ec370
Add Updated Memory Connector ADR and VectorDBRecordService inteface w…
westey-m 9083171
Fix syntax.
westey-m 5f968b3
Fix typos in adr.
westey-m 57cdabc
Update text from 3 to 4 responsibilities
westey-m ccbe297
Merge branch 'microsoft:main' into generic-vector-store-adr
westey-m 55531b1
Rename "rename" to "delete" since it more accurately reflects impact …
westey-m f997209
Remove datafield from vector attribute, since it's not required yet.
westey-m 9889cf1
Merge branch 'microsoft:main' into generic-vector-store-adr
westey-m 1058954
Update ADR with feedback.
westey-m 61823ab
Update execution plan with more detail.
westey-m b61d805
Address some feedback on the ADR around non-functional requirements a…
westey-m 9f43b51
Address pr comments.
westey-m 7ce6b09
Clarifying decorator based normalization cons as per PR comments.
westey-m 7c7ab63
Fix typo.
westey-m 8c4b4d0
Prefix attributes with MemoryRecord to reduce clashes.
westey-m 02f4322
Addressing ADR PR comments.
westey-m 303f698
Fix typo.
westey-m a27b268
Add notable method signature changes and comparison with other ai fra…
westey-m 4537a4c
Addming more PR comment updates to ADR doc.
westey-m File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
80 changes: 80 additions & 0 deletions
80
dotnet/src/SemanticKernel.Abstractions/Memory/IMemoryRecordService{TKey,TDataModel}.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,80 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System.Collections.Generic; | ||
using System.Diagnostics.CodeAnalysis; | ||
using System.Threading; | ||
using System.Threading.Tasks; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// An interface for adding, updating, deleting and retrieving records from a memory store. | ||
/// </summary> | ||
/// <typeparam name="TKey">The data type of the record key.</typeparam> | ||
/// <typeparam name="TDataModel">The data model to use for adding, updating and retrieving data from storage.</typeparam> | ||
[Experimental("SKEXP0001")] | ||
public interface IMemoryRecordService<TKey, TDataModel> | ||
where TDataModel : class | ||
{ | ||
/// <summary> | ||
/// Gets a memory record from the data store. Does not guarantee that the collection exists. | ||
/// Throws if the record is not found. | ||
/// </summary> | ||
/// <param name="key">The unique id associated with the memory record to get.</param> | ||
/// <param name="options">Optional options for retrieving the record.</param> | ||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param> | ||
/// <returns>The memory record if found, otherwise null.</returns> | ||
Task<TDataModel?> GetAsync(TKey key, GetRecordOptions? options = default, CancellationToken cancellationToken = default); | ||
|
||
/// <summary> | ||
/// Gets a batch of memory records from the data store. Does not guarantee that the collection exists. | ||
/// Throws if any of the records are not found. | ||
/// Gets will be made in a single request or in a single parallel batch depending on the available store functionality. | ||
/// </summary> | ||
/// <param name="keys">The unique ids associated with the memory record to get.</param> | ||
/// <param name="options">Optional options for retrieving the records.</param> | ||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param> | ||
/// <returns>The vecmemorytor records associated with the unique keys provided.</returns> | ||
IAsyncEnumerable<TDataModel?> GetBatchAsync(IEnumerable<TKey> keys, GetRecordOptions? options = default, CancellationToken cancellationToken = default); | ||
|
||
/// <summary> | ||
/// Deletes a memory record from the data store. Does not guarantee that the collection exists. | ||
/// </summary> | ||
/// <param name="key">The unique id associated with the memory record to remove.</param> | ||
/// <param name="options">Optional options for removing the record.</param> | ||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param> | ||
/// <returns>The unique identifier for the memory record.</returns> | ||
Task DeleteAsync(TKey key, DeleteRecordOptions? options = default, CancellationToken cancellationToken = default); | ||
|
||
/// <summary> | ||
/// Deletes a batch of memory records from the data store. Does not guarantee that the collection exists. | ||
/// Deletes will be made in a single request or in a single parallel batch depending on the available store functionality. | ||
/// </summary> | ||
/// <param name="keys">The unique ids associated with the memory records to remove.</param> | ||
/// <param name="options">Optional options for removing the records.</param> | ||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param> | ||
Task DeleteBatchAsync(IEnumerable<TKey> keys, DeleteRecordOptions? options = default, CancellationToken cancellationToken = default); | ||
|
||
/// <summary> | ||
/// Upserts a memory record into the data store. Does not guarantee that the collection exists. | ||
/// If the record already exists, it will be updated. | ||
/// If the record does not exist, it will be created. | ||
/// </summary> | ||
/// <param name="record">The memory record to upsert.</param> | ||
/// <param name="options">Optional options for upserting the record.</param> | ||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param> | ||
/// <returns>The unique identifier for the memory record.</returns> | ||
Task<TKey> UpsertAsync(TDataModel record, UpsertRecordOptions? options = default, CancellationToken cancellationToken = default); | ||
|
||
/// <summary> | ||
/// Upserts a group of memory records into the data store. Does not guarantee that the collection exists. | ||
/// If the record already exists, it will be updated. | ||
/// If the record does not exist, it will be created. | ||
/// Upserts will be made in a single request or in a single parallel batch depending on the available store functionality. | ||
/// </summary> | ||
/// <param name="records">The memory records to upsert.</param> | ||
/// <param name="options">Optional options for upserting the records.</param> | ||
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param> | ||
/// <returns>The unique identifiers for the memory records.</returns> | ||
IAsyncEnumerable<TKey> UpsertBatchAsync(IEnumerable<TDataModel> records, UpsertRecordOptions? options = default, CancellationToken cancellationToken = default); | ||
} |
25 changes: 25 additions & 0 deletions
25
...rc/SemanticKernel.Abstractions/Memory/MemoryRecordAttributes/MemoryRecordDataAttribute.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System; | ||
using System.Diagnostics.CodeAnalysis; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// Attribute to mark a property on a vector model class as the data that is being indexed. | ||
/// </summary> | ||
[Experimental("SKEXP0001")] | ||
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] | ||
public sealed class MemoryRecordDataAttribute : Attribute | ||
{ | ||
/// <summary> | ||
/// Gets or sets a value indicating whether this data field has an associated embedding field. | ||
/// </summary> | ||
/// <remarks>Defaults to <see langword="false" /></remarks> | ||
public bool HasEmbedding { get; init; } | ||
|
||
/// <summary> | ||
/// Gets or sets the name of the property that contains the embedding for this data field. | ||
/// </summary> | ||
public string? EmbeddingPropertyName { get; init; } | ||
} |
15 changes: 15 additions & 0 deletions
15
...src/SemanticKernel.Abstractions/Memory/MemoryRecordAttributes/MemoryRecordKeyAttribute.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System; | ||
using System.Diagnostics.CodeAnalysis; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// Attribute to mark a property on a class as the key under which data is stored in a vector store. | ||
/// </summary> | ||
[Experimental("SKEXP0001")] | ||
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] | ||
public sealed class MemoryRecordKeyAttribute : Attribute | ||
{ | ||
} |
15 changes: 15 additions & 0 deletions
15
.../SemanticKernel.Abstractions/Memory/MemoryRecordAttributes/MemoryRecordVectorAttribute.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System; | ||
using System.Diagnostics.CodeAnalysis; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// Attribute to mark a property on a vector model class as the vector. | ||
/// </summary> | ||
[Experimental("SKEXP0001")] | ||
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)] | ||
public sealed class MemoryRecordVectorAttribute : Attribute | ||
{ | ||
} |
33 changes: 33 additions & 0 deletions
33
dotnet/src/SemanticKernel.Abstractions/Memory/RecordOptions/DeleteRecordOptions.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System.Diagnostics.CodeAnalysis; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// Optional options when calling <see cref="IMemoryRecordService{TKey, TDataModel}.DeleteAsync"/>. | ||
/// </summary> | ||
[Experimental("SKEXP0001")] | ||
public class DeleteRecordOptions | ||
{ | ||
/// <summary> | ||
/// Initializes a new instance of the <see cref="DeleteRecordOptions"/> class. | ||
/// </summary> | ||
public DeleteRecordOptions() | ||
{ | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="DeleteRecordOptions"/> class by cloning the given options. | ||
/// </summary> | ||
/// <param name="source">The options to clone</param> | ||
public DeleteRecordOptions(DeleteRecordOptions source) | ||
{ | ||
this.CollectionName = source.CollectionName; | ||
} | ||
|
||
/// <summary> | ||
/// Get or sets an optional collection name to use for this operation that is different to the default. | ||
/// </summary> | ||
public string? CollectionName { get; init; } | ||
} |
39 changes: 39 additions & 0 deletions
39
dotnet/src/SemanticKernel.Abstractions/Memory/RecordOptions/GetRecordOptions.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,39 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System.Diagnostics.CodeAnalysis; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// Optional options when calling <see cref="IMemoryRecordService{TKey, TDataModel}.GetAsync"/>. | ||
/// </summary> | ||
[Experimental("SKEXP0001")] | ||
public class GetRecordOptions | ||
{ | ||
/// <summary> | ||
/// Initializes a new instance of the <see cref="GetRecordOptions"/> class. | ||
/// </summary> | ||
public GetRecordOptions() | ||
{ | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="GetRecordOptions"/> class by cloning the given options. | ||
/// </summary> | ||
/// <param name="source">The options to clone</param> | ||
public GetRecordOptions(GetRecordOptions source) | ||
{ | ||
this.CollectionName = source.CollectionName; | ||
this.IncludeVectors = source.IncludeVectors; | ||
} | ||
|
||
/// <summary> | ||
/// Get or sets an optional collection name to use for this operation that is different to the default. | ||
/// </summary> | ||
public string? CollectionName { get; init; } | ||
|
||
/// <summary> | ||
/// Get or sets a value indicating whether to include vectors in the retrieval result. | ||
/// </summary> | ||
public bool IncludeVectors { get; init; } = false; | ||
} |
33 changes: 33 additions & 0 deletions
33
dotnet/src/SemanticKernel.Abstractions/Memory/RecordOptions/UpsertRecordOptions.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
// Copyright (c) Microsoft. All rights reserved. | ||
|
||
using System.Diagnostics.CodeAnalysis; | ||
|
||
namespace Microsoft.SemanticKernel.Memory; | ||
|
||
/// <summary> | ||
/// Optional options when calling <see cref="IMemoryRecordService{TKey, TDataModel}.UpsertAsync"/>. | ||
/// </summary> | ||
[Experimental("SKEXP0001")] | ||
public class UpsertRecordOptions | ||
{ | ||
/// <summary> | ||
/// Initializes a new instance of the <see cref="UpsertRecordOptions"/> class. | ||
/// </summary> | ||
public UpsertRecordOptions() | ||
{ | ||
} | ||
|
||
/// <summary> | ||
/// Initializes a new instance of the <see cref="UpsertRecordOptions"/> class by cloning the given options. | ||
/// </summary> | ||
/// <param name="source">The options to clone</param> | ||
public UpsertRecordOptions(UpsertRecordOptions source) | ||
{ | ||
this.CollectionName = source.CollectionName; | ||
} | ||
|
||
/// <summary> | ||
/// Get or sets an optional collection name to use for this operation that is different to the default. | ||
/// </summary> | ||
public string? CollectionName { get; init; } | ||
} |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.