.Net: Getting Started with Memory

dotnet/SK-dotnet.sln

@@ -318,7 +318,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Redis.UnitTests"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connectors.Qdrant.UnitTests", "src\Connectors\Connectors.Qdrant.UnitTests\Connectors.Qdrant.UnitTests.csproj", "{E92AE954-8F3A-4A6F-A4F9-DC12017E5AAF}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "StepwisePlannerMigration", "samples\Demos\StepwisePlannerMigration\StepwisePlannerMigration.csproj", "{38374C62-0263-4FE8-A18C-70FC8132912B}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StepwisePlannerMigration", "samples\Demos\StepwisePlannerMigration\StepwisePlannerMigration.csproj", "{38374C62-0263-4FE8-A18C-70FC8132912B}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GettingStartedWithMemory", "samples\GettingStartedWithMemory\GettingStartedWithMemory.csproj", "{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -795,6 +797,12 @@ Global
 		{38374C62-0263-4FE8-A18C-70FC8132912B}.Publish|Any CPU.Build.0 = Debug|Any CPU
 		{38374C62-0263-4FE8-A18C-70FC8132912B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{38374C62-0263-4FE8-A18C-70FC8132912B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}.Publish|Any CPU.ActiveCfg = Debug|Any CPU
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}.Publish|Any CPU.Build.0 = Debug|Any CPU
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -904,6 +912,7 @@ Global
 		{1D4667B9-9381-4E32-895F-123B94253EE8} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
 		{E92AE954-8F3A-4A6F-A4F9-DC12017E5AAF} = {0247C2C9-86C3-45BA-8873-28B0948EDC0C}
 		{38374C62-0263-4FE8-A18C-70FC8132912B} = {5D4C0700-BBB5-418F-A7B2-F392B9A18263}
+		{E857766A-C9AC-4FE7-AB51-734BBE21A6B2} = {FA3720F1-C99A-49B2-9577-A940257098BF}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {FBDC56A3-86AD-4323-AA0F-201E59123B83}

dotnet/samples/Concepts/Memory/VectorStore_HelloWorld.cs

@@ -0,0 +1,97 @@
+// Copyright (c) Microsoft. All rights reserved.
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Connectors.Qdrant;
+using Microsoft.SemanticKernel.Connectors.Redis;
+using Microsoft.SemanticKernel.Data;
+
+namespace Memory;
+
+public class VectorStore_HelloWorld(ITestOutputHelper output) : BaseTest(output)
+{
+    [Fact]
+    public async Task UpsertGetVolatileAsync()
+    {
+        var kernel = Kernel
+            .CreateBuilder()
+            .AddVolatileVectorStore()
+            .Build();
+
+        var vectorStore = kernel.GetRequiredService<IVectorStore>();
+
+        var collection = vectorStore.GetCollection<string, StringKeyModel>("mycollection");
+
+        await collection.CreateCollectionIfNotExistsAsync();
+
+        var upsertedId = await collection.UpsertAsync(new StringKeyModel { Key = "key1", SomeStringData = "the string payload", Embedding = new ReadOnlyMemory<float>(new float[4] { 1.1f, 2.2f, 3.3f, 4.4f }) });
+        Console.WriteLine(upsertedId);
+
+        var retrievedData = await collection.GetAsync(upsertedId, new() { IncludeVectors = true });
+        Console.WriteLine(retrievedData);
+    }
+
+    [Fact]
+    public async Task UpsertGetQdrantAsync()
+    {
+        var kernel = Kernel
+            .CreateBuilder()
+            .AddQdrantVectorStore("localhost")
+            .Build();
+
+        var vectorStore = kernel.GetRequiredService<IVectorStore>();
+
+        var collection = vectorStore.GetCollection<Guid, GuidKeyModel>("mycollection");
+
+        await collection.CreateCollectionIfNotExistsAsync();
+
+        var upsertedId = await collection.UpsertAsync(new GuidKeyModel { Key = Guid.NewGuid(), SomeStringData = "the string payload", Embedding = new ReadOnlyMemory<float>(new float[4] { 1.1f, 2.2f, 3.3f, 4.4f }) });
+        Console.WriteLine(upsertedId);
+
+        var retrievedData = await collection.GetAsync(upsertedId, new() { IncludeVectors = true });
+        Console.WriteLine(retrievedData);
+    }
+
+    [Fact]
+    public async Task UpsertGetRedisAsync()
+    {
+        var kernel = Kernel
+            .CreateBuilder()
+            .AddRedisVectorStore("localhost:6379")
+            .Build();
+
+        var vectorStore = kernel.GetRequiredService<IVectorStore>();
+
+        var collection = vectorStore.GetCollection<string, StringKeyModel>("mycollection");
+
+        await collection.CreateCollectionIfNotExistsAsync();
+
+        var upsertedId = await collection.UpsertAsync(new StringKeyModel { Key = "key1", SomeStringData = "the string payload", Embedding = new ReadOnlyMemory<float>(new float[4] { 1.1f, 2.2f, 3.3f, 4.4f }) });
+        Console.WriteLine(upsertedId);
+
+        var retrievedData = await collection.GetAsync(upsertedId, new() { IncludeVectors = true });
+        Console.WriteLine(retrievedData);
+    }
+
+    private sealed class StringKeyModel
+    {
+        [VectorStoreRecordKey]
+        public string Key { get; set; }
+
+        [VectorStoreRecordData]
+        public string SomeStringData { get; set; }
+
+        [VectorStoreRecordVector(4)]
+        public ReadOnlyMemory<float> Embedding { get; set; }
+    }
+
+    private sealed class GuidKeyModel
+    {
+        [VectorStoreRecordKey]
+        public Guid Key { get; set; }
+
+        [VectorStoreRecordData]
+        public string SomeStringData { get; set; }
+
+        [VectorStoreRecordVector(4)]
+        public ReadOnlyMemory<float> Embedding { get; set; }
+    }
+}

dotnet/samples/GettingStartedWithMemory/GettingStartedWithMemory.csproj

@@ -0,0 +1,57 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <AssemblyName>GettingStartedWithMemory</AssemblyName>
+    <RootNamespace></RootNamespace>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <TargetFramework>net8.0</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <IsPackable>false</IsPackable>
+    <!-- Suppress: "Declare types in namespaces", "Require ConfigureAwait", "Experimental" -->
+    <NoWarn>$(NoWarn);CS8618,IDE0009,CA1051,CA1050,CA1707,CA1054,CA2007,VSTHRD111,CS1591,RCS1110,RCS1243,CA5394,SKEXP0001,SKEXP0010,SKEXP0020,SKEXP0040,SKEXP0050,SKEXP0060,SKEXP0070,SKEXP0101</NoWarn>
+    <OutputType>Library</OutputType>
+    <UserSecretsId>5ee045b0-aea3-4f08-8d31-32d1a6f8fed0</UserSecretsId>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xRetry" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.abstractions" />
+    <PackageReference Include="xunit.runner.visualstudio">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="Azure.Identity" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+    <PackageReference Include="Microsoft.Extensions.Http" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" />
+    <PackageReference Include="Microsoft.Extensions.Logging" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+    <PackageReference Include="SharpToken" />
+    <PackageReference Include="Microsoft.ML.Tokenizers" />
+    <PackageReference Include="Microsoft.DeepDev.TokenizerLib" />
+    <PackageReference Include="System.Linq.Async" />
+    <PackageReference Include="System.Numerics.Tensors" />
+  </ItemGroup>
+
+  <Import Project="$(RepoRoot)/dotnet/src/InternalUtilities/samples/SamplesInternalUtilities.props" />
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\Connectors\Connectors.OpenAI\Connectors.OpenAI.csproj" />
+    <ProjectReference Include="..\..\src\Extensions\PromptTemplates.Handlebars\PromptTemplates.Handlebars.csproj" />
+    <ProjectReference Include="..\..\src\Functions\Functions.OpenApi\Functions.OpenApi.csproj" />
+    <ProjectReference Include="..\..\src\Functions\Functions.Yaml\Functions.Yaml.csproj" />
+    <ProjectReference Include="..\..\src\SemanticKernel.Abstractions\SemanticKernel.Abstractions.csproj" />
+    <ProjectReference Include="..\..\src\SemanticKernel.Core\SemanticKernel.Core.csproj" />
+    <PackageReference Include="System.Text.Json" />
+  </ItemGroup>
+  <ItemGroup>
+    <Using Include="Xunit" />
+    <Using Include="Xunit.Abstractions"/>
+  </ItemGroup>
+</Project>

dotnet/samples/GettingStartedWithMemory/README.md

@@ -0,0 +1,64 @@
+# Getting Started With Semantic Kernel Memory Connector
+
+This project contains a step by step guide to get started with the Semantic Kernel Memory Connectors.
+
+The examples can be run as integration tests but their code can also be copied to stand-alone programs.
+
+1. Performing **C**reate, **R**ead, **U**pdate and **D**elete optations using the `VolatileVectorStore`
+2. Configure the Vector Store you are using, we currently support:
+   1. Azure AI Search
+   2. Redis
+   3. Qdrant
+   4. Pinecone
+3. Options to configure your data model and custom mapping
+4. ...
+
+## Configuring a Vector Database
+Here is a table showing the different databases plus useful information on how to get started with them. 
+
+- Docker Command: The docker command to run a local container with the database where available 
+- Portal: The portal where it�s possible to observe any collections and records created 
+- Fixture Name: Where available a fixture that can be used to automatically start and stop the docker container (alternative to running it manually) 
+- Add to Kernel: An example of the command to use to add the VectorStore to DI on the kernelbuilder. 
+
+
+| Database | Docker Command | Portal | Fixture Name | Add to Kernel |
+|----------|----------------|--------|--------------|---------------|
+| Redis | `docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest` | http://localhost:8001/redis-stack/browser | IClassFixture<VectorStoreRedisContainerFixture> | kernelBuilder.AddRedisVectorStore("localhost:6379"); |
+| Qdrant | `docker run -d --name qdrant -p 6333:6333 -p 6334:6334 qdrant/qdrant:latest` | http://localhost:6333/dashboard | IClassFixture<VectorStoreQdrantContainerFixture> | kernelBuilder.AddQdrantVectorStore("localhost"); |
+| Volatile | n/a | n/a | IClassFixture<VectorStoreVolatileFixture> | kernelBuilder.AddVolatileVectorStore(); |
+| Azure AI Search | n/a | n/a | IClassFixture<VectorStoreAzureAISearchFixture> | kernelBuilder.AddAzureAISearchVectorStore(new Uri("https://vectorstore-bugbash-2024-07-25.search.windows.net"), new AzureKeyCredential("")); |
+| Pinecone | n/a | Requires signup: https://www.pinecone.io/ (API key can be requested after signup) | n/a | kernelBuilder.AddPineconeVectorStore("api key"); |
+
+
+## Configuring Secrets
+
+Most of the examples will require secrets and credentials, to access OpenAI, Azure OpenAI,
+Bing and other resources. We suggest using .NET
+[Secret Manager](https://learn.microsoft.com/en-us/aspnet/core/security/app-secrets)
+to avoid the risk of leaking secrets into the repository, branches and pull requests.
+You can also use environment variables if you prefer.
+
+To set your secrets with Secret Manager:
+
+```
+cd dotnet/samples/Concepts
+
+dotnet user-secrets init
+
+dotnet user-secrets set "OpenAI:ModelId" "..."
+dotnet user-secrets set "OpenAI:ChatModelId" "..."
+dotnet user-secrets set "OpenAI:EmbeddingModelId" "..."
+dotnet user-secrets set "OpenAI:ApiKey" "..."
+
+```
+
+To set your secrets with environment variables, use these names:
+
+```
+# OpenAI
+OpenAI__ModelId
+OpenAI__ChatModelId
+OpenAI__EmbeddingModelId
+OpenAI__ApiKey
+```

dotnet/samples/GettingStartedWithMemory/Step1_VolatileVectorStore.cs

@@ -0,0 +1,86 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json;
+using Microsoft.SemanticKernel;
+using Microsoft.SemanticKernel.Data;
+using Microsoft.SemanticKernel.Embeddings;
+
+namespace GettingStarted;
+
+/// <summary>
+/// This example shows use a <see cref="IVectorStore"/>.
+/// </summary>
+public sealed class Step1_VolatileVectorStore(ITestOutputHelper output) : BaseTest(output)
+{
+    /// <summary>
+    /// Show how to create an <see cref="IVectorStoreRecordCollection{TKey, TRecord}"/> and use it to Create and Read a record.
+    /// </summary>
+    [Fact]
+    public async Task UpsertReadDeleteRecordInVectorStoreAsync()
+    {
+        var kernel = Kernel
+            .CreateBuilder()
+            .AddVolatileVectorStore()
+            .AddOpenAITextEmbeddingGeneration(
+                modelId: TestConfiguration.OpenAI.EmbeddingModelId,
+                apiKey: TestConfiguration.OpenAI.ApiKey
+            )
+            .Build();
+
+        var vectorStore = kernel.GetRequiredService<IVectorStore>();
+        var embeddingGeneration = kernel.GetRequiredService<ITextEmbeddingGenerationService>();
+
+        embeddingGeneration.GenerateEmbeddingAsync("Hello, world!").Wait();
+
+        var collection = vectorStore.GetCollection<string, MyModel>("MyCollection");
+
+        await collection.CreateCollectionIfNotExistsAsync();
+
+        // Generate an embedding for the sample data
+        var stringData = "Hello, world!";
+        var embedding = await embeddingGeneration.GenerateEmbeddingAsync(stringData);
+
+        // Upsert a record
+        var recordId = await collection.UpsertAsync(new MyModel
+        {
+            Key = Guid.NewGuid().ToString(),
+            StringData = stringData,
+            Embedding = embedding,
+        });
+        Console.WriteLine(recordId);
+
+        // Retrieve the record
+        var retrievedData = await collection.GetAsync(recordId, new() { IncludeVectors = true });
+        Console.WriteLine(retrievedData);
+
+        // Delete the record
+        await collection.DeleteAsync(recordId);
+
+        // Try to retrieve the record again
+        retrievedData = await collection.GetAsync(recordId, new() { IncludeVectors = true });
+        Console.WriteLine(retrievedData);
+    }
+
+    /// <summary>
+    /// Data model for the Hello World example.
+    /// </summary>
+    private sealed class MyModel
+    {
+        private readonly JsonSerializerOptions _jsonSerializerOptions = new()
+        {
+            WriteIndented = true,
+            Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        };
+
+        [VectorStoreRecordKey]
+        public string Key { get; set; }
+
+        [VectorStoreRecordData]
+        public string StringData { get; set; }
+
+        [VectorStoreRecordVector(4)]
+        public ReadOnlyMemory<float> Embedding { get; set; }
+
+        public override string ToString() => JsonSerializer.Serialize(this, _jsonSerializerOptions);
+    }
+}