.Net: Adding VectorStore abstractions and 5 sample implementations

dotnet/Directory.Packages.props

@@ -30,6 +30,7 @@
     <PackageVersion Include="Microsoft.Identity.Client" Version="4.61.3" />
     <PackageVersion Include="Microsoft.ML.OnnxRuntime" Version="1.18.1" />
     <PackageVersion Include="FastBertTokenizer" Version="1.0.28" />
+    <PackageVersion Include="Pinecone.NET" Version="2.1.1" />
     <PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
     <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
     <PackageVersion Include="System.Memory.Data" Version="8.0.0" />
@@ -68,6 +69,7 @@
     <PackageVersion Include="xunit.runner.visualstudio" Version="2.5.7" />
     <PackageVersion Include="xretry" Version="1.9.0" />
     <PackageVersion Include="coverlet.collector" Version="6.0.2" />
+    <PackageVersion Include="Docker.DotNet" Version="3.125.15" />
     <!-- Plugins -->
     <PackageVersion Include="DocumentFormat.OpenXml" Version="3.0.2" />
     <PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.2" />
@@ -92,6 +94,7 @@
     <PackageVersion Include="Milvus.Client" Version="2.3.0-preview.1" />
     <PackageVersion Include="Testcontainers.Milvus" Version="3.8.0" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.2.1" />
+    <PackageVersion Include="Qdrant.Client" Version="1.9.0" />
     <!-- Symbols -->
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
     <!-- Toolset -->

dotnet/samples/Concepts/Concepts.csproj

@@ -14,6 +14,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="Docker.DotNet" />
     <PackageReference Include="Microsoft.NET.Test.Sdk" />
     <PackageReference Include="xRetry" />
     <PackageReference Include="xunit" />

dotnet/samples/Concepts/Memory/VectorStoreFixtures/VectorStoreInfra.cs

@@ -0,0 +1,108 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Docker.DotNet;
+using Docker.DotNet.Models;
+
+namespace Memory.VectorStoreFixtures;
+
+/// <summary>
+/// Helper class that creates and deletes containers for the vector store examples.
+/// </summary>
+internal static class VectorStoreInfra
+{
+    /// <summary>
+    /// Setup the qdrant container by pulling the image and running it.
+    /// </summary>
+    /// <param name="client">The docker client to create the container with.</param>
+    /// <returns>The id of the container.</returns>
+    public static async Task<string> SetupQdrantContainerAsync(DockerClient client)
+    {
+        await client.Images.CreateImageAsync(
+            new ImagesCreateParameters
+            {
+                FromImage = "qdrant/qdrant",
+                Tag = "latest",
+            },
+            null,
+            new Progress<JSONMessage>());
+
+        var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
+        {
+            Image = "qdrant/qdrant",
+            HostConfig = new HostConfig()
+            {
+                PortBindings = new Dictionary<string, IList<PortBinding>>
+                {
+                    {"6333", new List<PortBinding> {new() {HostPort = "6333" } }},
+                    {"6334", new List<PortBinding> {new() {HostPort = "6334" } }}
+                },
+                PublishAllPorts = true
+            },
+            ExposedPorts = new Dictionary<string, EmptyStruct>
+            {
+                { "6333", default },
+                { "6334", default }
+            },
+        });
+
+        await client.Containers.StartContainerAsync(
+            container.ID,
+            new ContainerStartParameters());
+
+        return container.ID;
+    }
+
+    /// <summary>
+    /// Setup the redis container by pulling the image and running it.
+    /// </summary>
+    /// <param name="client">The docker client to create the container with.</param>
+    /// <returns>The id of the container.</returns>
+    public static async Task<string> SetupRedisContainerAsync(DockerClient client)
+    {
+        await client.Images.CreateImageAsync(
+            new ImagesCreateParameters
+            {
+                FromImage = "redis/redis-stack",
+                Tag = "latest",
+            },
+            null,
+            new Progress<JSONMessage>());
+
+        var container = await client.Containers.CreateContainerAsync(new CreateContainerParameters()
+        {
+            Image = "redis/redis-stack",
+            HostConfig = new HostConfig()
+            {
+                PortBindings = new Dictionary<string, IList<PortBinding>>
+                {
+                    {"6379", new List<PortBinding> {new() {HostPort = "6379"}}},
+                    {"8001", new List<PortBinding> {new() {HostPort = "8001"}}}
+                },
+                PublishAllPorts = true
+            },
+            ExposedPorts = new Dictionary<string, EmptyStruct>
+            {
+                { "6379", default },
+                { "8001", default }
+            },
+        });
+
+        await client.Containers.StartContainerAsync(
+            container.ID,
+            new ContainerStartParameters());
+
+        return container.ID;
+    }
+
+    /// <summary>
+    /// Stop and delete the container with the specified id.
+    /// </summary>
+    /// <param name="client">The docker client to delete the container in.</param>
+    /// <param name="containerId">The id of the container to delete.</param>
+    /// <returns>An async task.</returns>
+    public static async Task DeleteContainerAsync(DockerClient client, string containerId)
+    {
+        await client.Containers.StopContainerAsync(containerId, new ContainerStopParameters());
+        await client.Containers.RemoveContainerAsync(containerId, new ContainerRemoveParameters());
+    }
+}

dotnet/samples/Concepts/Memory/VectorStoreFixtures/VectorStoreQdrantContainerFixture.cs

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Docker.DotNet;
+using Qdrant.Client;
+
+namespace Memory.VectorStoreFixtures;
+
+/// <summary>
+/// Fixture to use for creating a Qdrant container before tests and delete it after tests.
+/// </summary>
+public class VectorStoreQdrantContainerFixture : IAsyncLifetime
+{
+    private DockerClient? _dockerClient;
+    private string? _qdrantContainerId;
+
+    public async Task InitializeAsync()
+    {
+    }
+
+    public async Task ManualInitializeAsync()
+    {
+        if (this._qdrantContainerId == null)
+        {
+            // Connect to docker and start the docker container.
+            using var dockerClientConfiguration = new DockerClientConfiguration();
+            this._dockerClient = dockerClientConfiguration.CreateClient();
+            this._qdrantContainerId = await VectorStoreInfra.SetupQdrantContainerAsync(this._dockerClient);
+
+            // Delay until the Qdrant server is ready.
+            var qdrantClient = new QdrantClient("localhost");
+            var succeeded = false;
+            var attemptCount = 0;
+            while (!succeeded && attemptCount++ < 10)
+            {
+                try
+                {
+                    await qdrantClient.ListCollectionsAsync();
+                    succeeded = true;
+                }
+                catch (Exception)
+                {
+                    await Task.Delay(1000);
+                }
+            }
+        }
+    }
+
+    public async Task DisposeAsync()
+    {
+        if (this._dockerClient != null && this._qdrantContainerId != null)
+        {
+            // Delete docker container.
+            await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._qdrantContainerId);
+        }
+    }
+}

dotnet/samples/Concepts/Memory/VectorStoreFixtures/VectorStoreRedisContainerFixture.cs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Docker.DotNet;
+
+namespace Memory.VectorStoreFixtures;
+
+/// <summary>
+/// Fixture to use for creating a Redis container before tests and delete it after tests.
+/// </summary>
+public class VectorStoreRedisContainerFixture : IAsyncLifetime
+{
+    private DockerClient? _dockerClient;
+    private string? _redisContainerId;
+
+    public async Task InitializeAsync()
+    {
+    }
+
+    public async Task ManualInitializeAsync()
+    {
+        if (this._redisContainerId == null)
+        {
+            // Connect to docker and start the docker container.
+            using var dockerClientConfiguration = new DockerClientConfiguration();
+            this._dockerClient = dockerClientConfiguration.CreateClient();
+            this._redisContainerId = await VectorStoreInfra.SetupRedisContainerAsync(this._dockerClient);
+        }
+    }
+
+    public async Task DisposeAsync()
+    {
+        if (this._dockerClient != null && this._redisContainerId != null)
+        {
+            // Delete docker container.
+            await VectorStoreInfra.DeleteContainerAsync(this._dockerClient, this._redisContainerId);
+        }
+    }
+}