Skip to content

.Net: Add Updated Memory Connector ADR and IMemoryRecordService interface with related classes. #6364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
815 changes: 815 additions & 0 deletions docs/decisions/0045-updated-vector-store-design.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Threading;
using System.Threading.Tasks;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// An interface for adding, updating, deleting and retrieving records from a memory store.
/// </summary>
/// <typeparam name="TKey">The data type of the record key.</typeparam>
/// <typeparam name="TDataModel">The data model to use for adding, updating and retrieving data from storage.</typeparam>
[Experimental("SKEXP0001")]
public interface IMemoryRecordService<TKey, TDataModel>
where TDataModel : class
{
/// <summary>
/// Gets a memory record from the data store. Does not guarantee that the collection exists.
/// Throws if the record is not found.
/// </summary>
/// <param name="key">The unique id associated with the memory record to get.</param>
/// <param name="options">Optional options for retrieving the record.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The memory record if found, otherwise null.</returns>
Task<TDataModel?> GetAsync(TKey key, GetRecordOptions? options = default, CancellationToken cancellationToken = default);

/// <summary>
/// Gets a batch of memory records from the data store. Does not guarantee that the collection exists.
/// Throws if any of the records are not found.
/// Gets will be made in a single request or in a single parallel batch depending on the available store functionality.
/// </summary>
/// <param name="keys">The unique ids associated with the memory record to get.</param>
/// <param name="options">Optional options for retrieving the records.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The vecmemorytor records associated with the unique keys provided.</returns>
IAsyncEnumerable<TDataModel?> GetBatchAsync(IEnumerable<TKey> keys, GetRecordOptions? options = default, CancellationToken cancellationToken = default);

/// <summary>
/// Deletes a memory record from the data store. Does not guarantee that the collection exists.
/// </summary>
/// <param name="key">The unique id associated with the memory record to remove.</param>
/// <param name="options">Optional options for removing the record.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The unique identifier for the memory record.</returns>
Task DeleteAsync(TKey key, DeleteRecordOptions? options = default, CancellationToken cancellationToken = default);

/// <summary>
/// Deletes a batch of memory records from the data store. Does not guarantee that the collection exists.
/// Deletes will be made in a single request or in a single parallel batch depending on the available store functionality.
/// </summary>
/// <param name="keys">The unique ids associated with the memory records to remove.</param>
/// <param name="options">Optional options for removing the records.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
Task DeleteBatchAsync(IEnumerable<TKey> keys, DeleteRecordOptions? options = default, CancellationToken cancellationToken = default);

/// <summary>
/// Upserts a memory record into the data store. Does not guarantee that the collection exists.
/// If the record already exists, it will be updated.
/// If the record does not exist, it will be created.
/// </summary>
/// <param name="record">The memory record to upsert.</param>
/// <param name="options">Optional options for upserting the record.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The unique identifier for the memory record.</returns>
Task<TKey> UpsertAsync(TDataModel record, UpsertRecordOptions? options = default, CancellationToken cancellationToken = default);

/// <summary>
/// Upserts a group of memory records into the data store. Does not guarantee that the collection exists.
/// If the record already exists, it will be updated.
/// If the record does not exist, it will be created.
/// Upserts will be made in a single request or in a single parallel batch depending on the available store functionality.
/// </summary>
/// <param name="records">The memory records to upsert.</param>
/// <param name="options">Optional options for upserting the records.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to monitor for cancellation requests. The default is <see cref="CancellationToken.None"/>.</param>
/// <returns>The unique identifiers for the memory records.</returns>
IAsyncEnumerable<TKey> UpsertBatchAsync(IEnumerable<TDataModel> records, UpsertRecordOptions? options = default, CancellationToken cancellationToken = default);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Diagnostics.CodeAnalysis;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// Attribute to mark a property on a vector model class as the data that is being indexed.
/// </summary>
[Experimental("SKEXP0001")]
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class DataAttribute : Attribute
{
/// <summary>
/// Gets or sets a value indicating whether this data field has an associated embedding field.
/// </summary>
/// <remarks>Defaults to <see langword="false" /></remarks>
public bool HasEmbedding { get; init; }

/// <summary>
/// Gets or sets the name of the property that contains the embedding for this data field.
/// </summary>
public string? EmbeddingPropertyName { get; init; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Diagnostics.CodeAnalysis;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// Attribute to mark a property on a class as the key under which data is stored in a vector store.
/// </summary>
[Experimental("SKEXP0001")]
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class KeyAttribute : Attribute
{
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// Copyright (c) Microsoft. All rights reserved.

using System;
using System.Diagnostics.CodeAnalysis;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// Attribute to mark a property on a vector model class as the vector.
/// </summary>
[Experimental("SKEXP0001")]
[AttributeUsage(AttributeTargets.Property, AllowMultiple = false)]
public sealed class VectorAttribute : Attribute
{
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Diagnostics.CodeAnalysis;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// Optional options when calling <see cref="IMemoryRecordService{TKey, TDataModel}.DeleteAsync"/>.
/// </summary>
[Experimental("SKEXP0001")]
public class DeleteRecordOptions
{
/// <summary>
/// Initializes a new instance of the <see cref="DeleteRecordOptions"/> class.
/// </summary>
public DeleteRecordOptions()
{
}

/// <summary>
/// Initializes a new instance of the <see cref="DeleteRecordOptions"/> class by cloning the given options.
/// </summary>
/// <param name="source">The options to clone</param>
public DeleteRecordOptions(DeleteRecordOptions source)
{
this.CollectionName = source.CollectionName;
}

/// <summary>
/// Get or sets an optional collection name to use for this operation that is different to the default.
/// </summary>
public string? CollectionName { get; init; }
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Diagnostics.CodeAnalysis;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// Optional options when calling <see cref="IMemoryRecordService{TKey, TDataModel}.GetAsync"/>.
/// </summary>
[Experimental("SKEXP0001")]
public class GetRecordOptions
{
/// <summary>
/// Initializes a new instance of the <see cref="GetRecordOptions"/> class.
/// </summary>
public GetRecordOptions()
{
}

/// <summary>
/// Initializes a new instance of the <see cref="GetRecordOptions"/> class by cloning the given options.
/// </summary>
/// <param name="source">The options to clone</param>
public GetRecordOptions(GetRecordOptions source)
{
this.CollectionName = source.CollectionName;
this.IncludeVectors = source.IncludeVectors;
}

/// <summary>
/// Get or sets an optional collection name to use for this operation that is different to the default.
/// </summary>
public string? CollectionName { get; init; }

/// <summary>
/// Get or sets a value indicating whether to include vectors in the retrieval result.
/// </summary>
public bool IncludeVectors { get; init; } = false;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright (c) Microsoft. All rights reserved.

using System.Diagnostics.CodeAnalysis;

namespace Microsoft.SemanticKernel.Memory;

/// <summary>
/// Optional options when calling <see cref="IMemoryRecordService{TKey, TDataModel}.UpsertAsync"/>.
/// </summary>
[Experimental("SKEXP0001")]
public class UpsertRecordOptions
{
/// <summary>
/// Initializes a new instance of the <see cref="UpsertRecordOptions"/> class.
/// </summary>
public UpsertRecordOptions()
{
}

/// <summary>
/// Initializes a new instance of the <see cref="UpsertRecordOptions"/> class by cloning the given options.
/// </summary>
/// <param name="source">The options to clone</param>
public UpsertRecordOptions(UpsertRecordOptions source)
{
this.CollectionName = source.CollectionName;
}

/// <summary>
/// Get or sets an optional collection name to use for this operation that is different to the default.
/// </summary>
public string? CollectionName { get; init; }
}
Loading