diff --git a/docs/decisions/0045-kernel-content-graduation.md b/docs/decisions/0045-kernel-content-graduation.md new file mode 100644 index 000000000000..43518ddfa2d3 --- /dev/null +++ b/docs/decisions/0045-kernel-content-graduation.md @@ -0,0 +1,437 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +contact: rogerbarreto +date: 2024-05-02 +deciders: rogerbarreto, markwallace-microsoft, sergeymenkshi, dmytrostruk, sergeymenshik, westey-m, matthewbolanos +consulted: stephentoub +--- + +# Kernel Content Types Graduation + +## Context and Problem Statement + +Currently, we have many Content Types in experimental state and this ADR will give some options on how to graduate them to stable state. + +## Decision Drivers + +- No breaking changes +- Simple approach, minimal complexity +- Allow extensibility +- Concise and clear + +## BinaryContent Graduation + +This content should be by content specializations or directly for types that aren't specific, similar to "application/octet-stream" mime type. + +> **Application/Octet-Stream** is the MIME used for arbitrary binary data or a stream of bytes that doesn't fit any other more specific MIME type. This MIME type is often used as a default or fallback type, indicating that the file should be treated as pure binary data. + +#### Current + +```csharp +public class BinaryContent : KernelContent +{ + public ReadOnlyMemory? Content { get; set; } + public async Task GetStreamAsync() + public async Task> GetContentAsync() + + ctor(ReadOnlyMemory? content = null) + ctor(Func> streamProvider) +} +``` + +#### Proposed + +```csharp +public class BinaryContent : KernelContent +{ + ReadOnlyMemory? Data { get; set; } + Uri? Uri { get; set; } + string DataUri { get; set; } + + bool CanRead { get; } // Indicates if the content can be read as bytes or data uri + + ctor(Uri? referencedUri) + ctor(string dataUri) + // MimeType is not optional but nullable to encourage this information to be passed always when available. + ctor(ReadOnlyMemory data, string? mimeType) + ctor() // Empty ctor for serialization scenarios +} +``` + +- No Content property (Avoid clashing and/or misleading information if used from a specialized type context) + + i.e: + + - `PdfContent.Content` (Describe the text only information) + - `PictureContent.Content` (Exposes a `Picture` type) + +- Move away from deferred (lazy loaded) content providers, simpler API. +- `GetContentAsync` removal (No more derrefed APIs) +- Added `Data` property as setter and getter for byte array content information. + + Setting this property will override the `DataUri` base64 data part. + +- Added `DataUri` property as setter and getter for data uri content information. + + Setting this property will override the `Data` and `MimeType` properties with the current payload details. + +- Add `Uri` property for referenced content information. This property is does not accept not a `UriData` and only supports non-data schemes. +- Add `CanRead` property (To indicate if the content can be read using `Data` or `DataUri` properties.) +- Dedicated constructors for Uri, DataUri and ByteArray + MimeType creation. + +Pros: + +- With no deferred content we have simpler API and a single responsibility for contents. +- Can be written and read in both `Data` or `DataUri` formats. +- Can have a `Uri` reference property, which is common for specialized contexts. +- Fully serializeable. +- Data Uri parameters support (serialization included). +- Data Uri and Base64 validation checks +- Data Uri and Data can be dynamically generated +- `CanRead` will clearly identify if the content can be read as `bytes` or `DataUri`. + +Cons: + +- Breaking change for experimental `BinaryContent` consumers + +### Data Uri Parameters + +According to [RFC 2397](https://datatracker.ietf.org/doc/html/rfc2397), the data uri scheme supports parameters + +Every parameter imported from the data uri will be added to the Metadata dictionary with the "data-uri-parameter-name" as key and its respetive value. + +#### Providing a parameterized data uri will include those parameters in the Metadata dictionary. + +```csharp +var content = new BinaryContent("data:application/json;parameter1=value1;parameter2=value2;base64,SGVsbG8gV29ybGQ="); +var parameter1 = content.Metadata["data-uri-parameter1"]; // value1 +var parameter2 = content.Metadata["data-uri-parameter2"]; // value2 +``` + +#### Deserialization of contents will also include those parameters when getting the DataUri property. + +```csharp +var json = """ +{ + "metadata": + { + "data-uri-parameter1":"value1", + "data-uri-parameter2":"value2" + }, + "mimeType":"application/json", + "data":"SGVsbG8gV29ybGQ=" +} +"""; +var content = JsonSerializer.Deserialize(json); +content.DataUri // "data:application/json;parameter1=value1;parameter2=value2;base64,SGVsbG8gV29ybGQ=" +``` + +### Specialization Examples + +#### ImageContent + +```csharp +public class ImageContent : BinaryContent +{ + ctor(Uri uri) : base(uri) + ctor(string dataUri) : base(dataUri) + ctor(ReadOnlyMemory data, string? mimeType) : base(data, mimeType) + ctor() // serialization scenarios +} + +public class AudioContent : BinaryContent +{ + ctor(Uri uri) +} +``` + +Pros: + +- Supports data uri large contents +- Allows a binary ImageContent to be created using dataUrl scheme and also be referenced by a Url. +- Supports Data Uri validation + +## ImageContent Graduation + +⚠️ Currently this is not experimental, breaking changes needed to be graduated to stable state with potential benefits. + +### Problems + +1. Current `ImageContent` does not derive from `BinaryContent` +2. Has an undesirable behavior allowing the same instance to have distinct `DataUri` and `Data` at the same time. +3. `Uri` property is used for both data uri and referenced uri information +4. `Uri` does not support large language data uri formats. +5. Not clear to the `sk developer` whenever the content is readable or not. + +#### Current + +```csharp +public class ImageContent : KernelContent +{ + Uri? Uri { get; set; } + public ReadOnlyMemory? Data { get; set; } + + ctor(ReadOnlyMemory? data) + ctor(Uri uri) + ctor() +} +``` + +#### Proposed + +As already shown in the `BinaryContent` section examples, the `ImageContent` can be graduated to be a `BinaryContent` specialization an inherit all the benefits it brings. + +```csharp +public class ImageContent : BinaryContent +{ + ctor(Uri uri) : base(uri) + ctor(string dataUri) : base(dataUri) + ctor(ReadOnlyMemory data, string? mimeType) : base(data, mimeType) + ctor() // serialization scenarios +} +``` + +Pros: + +- Can be used as a `BinaryContent` type +- Can be written and read in both `Data` or `DataUri` formats. +- Can have a `Uri` dedicated for referenced location. +- Fully serializeable. +- Data Uri parameters support (serialization included). +- Data Uri and Base64 validation checks +- Can be retrieved +- Data Uri and Data can be dynamically generated +- `CanRead` will clearly identify if the content can be read as `bytes` or `DataUri`. + +Cons: + +- ⚠️ Breaking change for `ImageContent` consumers + +### ImageContent Breaking Changes + +- `Uri` property will be dedicated solely for referenced locations (non-data-uri), attempting to add a `data-uri` format will throw an exception suggesting the usage of the `DataUri` property instead. +- Setting `DataUri` will override the `Data` and `MimeType` properties according with the information provided. +- Attempting to set an invalid `DataUri` will throw an exception. +- Setting `Data` will now override the `DataUri` data part. +- Attempting to serialize an `ImageContent` with data-uri in the `Uri` property will throw an exception. + +## AudioContent Graduation + +Similar to `ImageContent` proposal `AudioContent` can be graduated to be a `BinaryContent`. + +#### Current + +1. Current `AudioContent` does not derive support `Uri` referenced location +2. `Uri` property is used for both data uri and referenced uri information +3. `Uri` does not support large language data uri formats. +4. Not clear to the `sk developer` whenever the content is readable or not. + +```csharp +public class AudioContent : KernelContent +{ + public ReadOnlyMemory? Data { get; set; } + + ctor(ReadOnlyMemory? data) + ctor() +} +``` + +#### Proposed + +```csharp +public class AudioContent : BinaryContent +{ + ctor(Uri uri) : base(uri) + ctor(string dataUri) : base(dataUri) + ctor(ReadOnlyMemory data, string? mimeType) : base(data, mimeType) + ctor() // serialization scenarios +} +``` + +Pros: + +- Can be used as a `BinaryContent` type +- Can be written and read in both `Data` or `DataUri` formats. +- Can have a `Uri` dedicated for referenced location. +- Fully serializeable. +- Data Uri parameters support (serialization included). +- Data Uri and Base64 validation checks +- Can be retrieved +- Data Uri and Data can be dynamically generated +- `CanRead` will clearly identify if the content can be read as `bytes` or `DataUri`. + +Cons: + +- Experimental breaking change for `AudioContent` consumers + +## FunctionCallContent Graduation + +### Current + +No changes needed to current structure. + +Potentially we could have a base `FunctionContent` but at the same time is good having those two deriving from `KernelContent` providing a clear separation of concerns. + +```csharp +public sealed class FunctionCallContent : KernelContent +{ + public string? Id { get; } + public string? PluginName { get; } + public string FunctionName { get; } + public KernelArguments? Arguments { get; } + public Exception? Exception { get; init; } + + ctor(string functionName, string? pluginName = null, string? id = null, KernelArguments? arguments = null) + + public async Task InvokeAsync(Kernel kernel, CancellationToken cancellationToken = default) + public static IEnumerable GetFunctionCalls(ChatMessageContent messageContent) +} +``` + +## FunctionResultContent Graduation + +It may require some changes although the current structure is good. + +### Current + +- From a purity perspective the `Id` property can lead to confusion as it's not a response Id but a function call Id. +- ctors have different `functionCall` and `functionCallContent` parameter names for same type. + +```csharp +public sealed class FunctionResultContent : KernelContent +{ + public string? Id { get; } + public string? PluginName { get; } + public string? FunctionName { get; } + public object? Result { get; } + + ctor(string? functionName = null, string? pluginName = null, string? id = null, object? result = null) + ctor(FunctionCallContent functionCall, object? result = null) + ctor(FunctionCallContent functionCallContent, FunctionResult result) +} +``` + +### Proposed - Option 1 + +- Rename `Id` to `CallId` to avoid confusion. +- Adjust `ctor` parameters names. + +```csharp +public sealed class FunctionResultContent : KernelContent +{ + public string? CallId { get; } + public string? PluginName { get; } + public string? FunctionName { get; } + public object? Result { get; } + + ctor(string? functionName = null, string? pluginName = null, string? callId = null, object? result = null) + ctor(FunctionCallContent functionCallContent, object? result = null) + ctor(FunctionCallContent functionCallContent, FunctionResult functionResult) +} +``` + +### Proposed - Option 2 + +Use composition a have a dedicated CallContent within the `FunctionResultContent`. + +Pros: + +- `CallContent` has options to invoke a function again from its response which can be handy for some scenarios +- Brings clarity from where the result came from and what is result specific data (root class). +- Knowledge about the arguments used in the call. + +Cons: + +- Introduce one extra hop to get the `call` details from the result. + +```csharp +public sealed class FunctionResultContent : KernelContent +{ + public FunctionCallContent CallContent { get; } + public object? Result { get; } + + ctor(FunctionCallContent functionCallContent, object? result = null) + ctor(FunctionCallContent functionCallContent, FunctionResult functionResult) +} +``` + +## FileReferenceContent + AnnotationContent + +Those two contents were added to `SemanticKernel.Abstractions` due to Serialization convenience but are very specific to **OpenAI Assistant API** and should be kept as Experimental for now. + +As a graduation those should be into `SemanticKernel.Agents.OpenAI` following the suggestion below. + +```csharp +#pragma warning disable SKEXP0110 +[JsonDerivedType(typeof(AnnotationContent), typeDiscriminator: nameof(AnnotationContent))] +[JsonDerivedType(typeof(FileReferenceContent), typeDiscriminator: nameof(FileReferenceContent))] +#pragma warning disable SKEXP0110 +public abstract class KernelContent { ... } +``` + +This coupling should not be encouraged for other packages that have `KernelContent` specializations. + +### Solution - Usage of [JsonConverter](https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/converters-how-to?pivots=dotnet-6-0#registration-sample---jsonconverter-on-a-type) Annotations + +Creation of a dedicated `JsonConverter` helper into the `Agents.OpenAI` project to handle the serialization and deserialization of those types. + +Annotate those Content types with `[JsonConverter(typeof(KernelContentConverter))]` attribute to indicate the `JsonConverter` to be used. + +### Agents.OpenAI's JsonConverter Example + +```csharp +public class KernelContentConverter : JsonConverter +{ + public override KernelContent Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + using (var jsonDoc = JsonDocument.ParseValue(ref reader)) + { + var root = jsonDoc.RootElement; + var typeDiscriminator = root.GetProperty("TypeDiscriminator").GetString(); + switch (typeDiscriminator) + { + case nameof(AnnotationContent): + return JsonSerializer.Deserialize(root.GetRawText(), options); + case nameof(FileReferenceContent): + return JsonSerializer.Deserialize(root.GetRawText(), options); + default: + throw new NotSupportedException($"Type discriminator '{typeDiscriminator}' is not supported."); + } + } + } + + public override void Write(Utf8JsonWriter writer, KernelContent value, JsonSerializerOptions options) + { + JsonSerializer.Serialize(writer, value, value.GetType(), options); + } +} + +[JsonConverter(typeof(KernelContentConverter))] +public class FileReferenceContent : KernelContent +{ + public string FileId { get; init; } = string.Empty; + ctor() + ctor(string fileId, ...) +} + +[JsonConverter(typeof(KernelContentConverter))] +public class AnnotationContent : KernelContent +{ + public string? FileId { get; init; } + public string? Quote { get; init; } + public int StartIndex { get; init; } + public int EndIndex { get; init; } + public ctor() + public ctor(...) +} +``` + +## Decision Outcome + +- `BinaryContent`: Accepted. +- `ImageContent`: Breaking change accepted with benefits using the `BinaryContent` specialization. No backwards compatibility as the current `ImageContent` behavior is undesirable. +- `AudioContent`: Experimental breaking changes using the `BinaryContent` specialization. +- `FunctionCallContent`: Graduate as is. +- `FunctionResultContent`: Experimental breaking change from property `Id` to `CallId` to avoid confusion regarding being a function call Id or a response id. +- `FileReferenceContent` and `AnnotationContent`: No changes, continue as experimental. diff --git a/dotnet/samples/Concepts/Agents/Legacy_AgentCharts.cs b/dotnet/samples/Concepts/Agents/Legacy_AgentCharts.cs index 63143154ae63..877ba0971710 100644 --- a/dotnet/samples/Concepts/Agents/Legacy_AgentCharts.cs +++ b/dotnet/samples/Concepts/Agents/Legacy_AgentCharts.cs @@ -71,10 +71,9 @@ async Task InvokeAgentAsync(IAgentThread thread, string imageName, string questi var path = Path.Combine(Environment.CurrentDirectory, filename); Console.WriteLine($"# {message.Role}: {message.Content}"); Console.WriteLine($"# {message.Role}: {path}"); - var content = fileService.GetFileContent(message.Content); + var content = await fileService.GetFileContentAsync(message.Content); await using var outputStream = File.OpenWrite(filename); - await using var inputStream = await content.GetStreamAsync(); - await inputStream.CopyToAsync(outputStream); + await outputStream.WriteAsync(content.Data!.Value); Process.Start( new ProcessStartInfo { diff --git a/dotnet/samples/Concepts/Agents/Legacy_AgentTools.cs b/dotnet/samples/Concepts/Agents/Legacy_AgentTools.cs index acacc1ecc2fd..66d93ecc88d9 100644 --- a/dotnet/samples/Concepts/Agents/Legacy_AgentTools.cs +++ b/dotnet/samples/Concepts/Agents/Legacy_AgentTools.cs @@ -83,7 +83,7 @@ public async Task RunRetrievalToolAsync() var fileService = kernel.GetRequiredService(); var result = await fileService.UploadContentAsync( - new BinaryContent(() => Task.FromResult(EmbeddedResource.ReadStream("travelinfo.txt")!)), + new BinaryContent(await EmbeddedResource.ReadAllAsync("travelinfo.txt")!, "text/plain"), new OpenAIFileUploadExecutionSettings("travelinfo.txt", OpenAIFilePurpose.Assistants)); var fileId = result.Id; diff --git a/dotnet/samples/Concepts/Agents/OpenAIAssistant_MultipleContents.cs b/dotnet/samples/Concepts/Agents/OpenAIAssistant_MultipleContents.cs new file mode 100644 index 000000000000..49f36edce0f4 --- /dev/null +++ b/dotnet/samples/Concepts/Agents/OpenAIAssistant_MultipleContents.cs @@ -0,0 +1,146 @@ +// Copyright (c) Microsoft. All rights reserved. +using Azure.AI.OpenAI.Assistants; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Agents; +using Microsoft.SemanticKernel.Agents.OpenAI; +using Microsoft.SemanticKernel.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.OpenAI; +using Resources; + +namespace Agents; + +/// +/// Demonstrate using retrieval on . +/// +public class OpenAIAssistant_MultipleContents(ITestOutputHelper output) : BaseTest(output) +{ + /// + /// Retrieval tool not supported on Azure OpenAI. + /// + protected override bool ForceOpenAI => true; + + [Fact] + public async Task RunAsync() + { + OpenAIFileService fileService = new(TestConfiguration.OpenAI.ApiKey); + + BinaryContent[] files = [ + // Audio is not supported by Assistant API + // new AudioContent(await EmbeddedResource.ReadAllAsync("test_audio.wav")!, mimeType:"audio/wav", innerContent: "test_audio.wav"), + new ImageContent(await EmbeddedResource.ReadAllAsync("sample_image.jpg")!, mimeType: "image/jpeg") { InnerContent = "sample_image.jpg" }, + new ImageContent(await EmbeddedResource.ReadAllAsync("test_image.jpg")!, mimeType: "image/jpeg") { InnerContent = "test_image.jpg" }, + new BinaryContent(data: await EmbeddedResource.ReadAllAsync("travelinfo.txt"), mimeType: "text/plain") + { + InnerContent = "travelinfo.txt" + } + ]; + + var fileIds = new List(); + foreach (var file in files) + { + try + { + var uploadFile = await fileService.UploadContentAsync(file, + new OpenAIFileUploadExecutionSettings(file.InnerContent!.ToString()!, Microsoft.SemanticKernel.Connectors.OpenAI.OpenAIFilePurpose.Assistants)); + + fileIds.Add(uploadFile.Id); + } + catch (HttpOperationException hex) + { + Console.WriteLine(hex.ResponseContent); + Assert.Fail($"Failed to upload file: {hex.Message}"); + } + } + + // Define the agent + OpenAIAssistantAgent agent = + await OpenAIAssistantAgent.CreateAsync( + kernel: new(), + config: new(this.ApiKey, this.Endpoint), + new() + { + EnableRetrieval = true, // Enable retrieval + ModelId = this.Model, + // FileIds = fileIds Currently Assistant API only supports text files, no images or audio. + FileIds = [fileIds.Last()] + }); + + // Create a chat for agent interaction. + var chat = new AgentGroupChat(); + + // Respond to user input + try + { + await InvokeAgentAsync("Where did sam go?"); + await InvokeAgentAsync("When does the flight leave Seattle?"); + await InvokeAgentAsync("What is the hotel contact info at the destination?"); + } + finally + { + await agent.DeleteAsync(); + } + + // Local function to invoke agent and display the conversation messages. + async Task InvokeAgentAsync(string input) + { + chat.AddChatMessage(new ChatMessageContent(AuthorRole.User, input)); + + Console.WriteLine($"# {AuthorRole.User}: '{input}'"); + + await foreach (var content in chat.InvokeAsync(agent)) + { + Console.WriteLine($"# {content.Role} - {content.AuthorName ?? "*"}: '{content.Content}'"); + } + } + } + + [Fact] + public async Task SendingAndRetrievingFilesAsync() + { + var openAIClient = new AssistantsClient(TestConfiguration.OpenAI.ApiKey); + OpenAIFileService fileService = new(TestConfiguration.OpenAI.ApiKey); + + BinaryContent[] files = [ + new AudioContent(await EmbeddedResource.ReadAllAsync("test_audio.wav")!, mimeType: "audio/wav") { InnerContent = "test_audio.wav" }, + new ImageContent(await EmbeddedResource.ReadAllAsync("sample_image.jpg")!, mimeType: "image/jpeg") { InnerContent = "sample_image.jpg" }, + new ImageContent(await EmbeddedResource.ReadAllAsync("test_image.jpg")!, mimeType: "image/jpeg") { InnerContent = "test_image.jpg" }, + new BinaryContent(data: await EmbeddedResource.ReadAllAsync("travelinfo.txt"), mimeType: "text/plain") { InnerContent = "travelinfo.txt" } + ]; + + var fileIds = new Dictionary(); + foreach (var file in files) + { + var result = await openAIClient.UploadFileAsync(new BinaryData(file.Data), Azure.AI.OpenAI.Assistants.OpenAIFilePurpose.FineTune); + fileIds.Add(result.Value.Id, file); + } + + foreach (var file in (await openAIClient.GetFilesAsync(Azure.AI.OpenAI.Assistants.OpenAIFilePurpose.FineTune)).Value) + { + if (!fileIds.ContainsKey(file.Id)) + { + continue; + } + + var data = (await openAIClient.GetFileContentAsync(file.Id)).Value; + + var mimeType = fileIds[file.Id].MimeType; + var fileName = fileIds[file.Id].InnerContent!.ToString(); + var metadata = new Dictionary { ["id"] = file.Id }; + var uri = new Uri($"https://api.openai.com/v1/files/{file.Id}/content"); + var content = mimeType switch + { + "image/jpeg" => new ImageContent(data, mimeType) { Uri = uri, InnerContent = fileName, Metadata = metadata }, + "audio/wav" => new AudioContent(data, mimeType) { Uri = uri, InnerContent = fileName, Metadata = metadata }, + _ => new BinaryContent(data, mimeType) { Uri = uri, InnerContent = fileName, Metadata = metadata } + }; + + Console.WriteLine($"File: {fileName} - {mimeType}"); + + // Images tostring are different from the graduated contents for retrocompatibility + Console.WriteLine(content.ToString()); + + // Delete the test file remotely + await openAIClient.DeleteFileAsync(file.Id); + } + } +} diff --git a/dotnet/samples/Concepts/Agents/OpenAIAssistant_Retrieval.cs b/dotnet/samples/Concepts/Agents/OpenAIAssistant_Retrieval.cs index f189bfbba937..2df655d07630 100644 --- a/dotnet/samples/Concepts/Agents/OpenAIAssistant_Retrieval.cs +++ b/dotnet/samples/Concepts/Agents/OpenAIAssistant_Retrieval.cs @@ -24,12 +24,9 @@ public async Task RunAsync() OpenAIFileService fileService = new(TestConfiguration.OpenAI.ApiKey); OpenAIFileReference uploadFile = - await fileService.UploadContentAsync( - new BinaryContent(() => Task.FromResult(EmbeddedResource.ReadStream("travelinfo.txt")!)), + await fileService.UploadContentAsync(new BinaryContent(await EmbeddedResource.ReadAllAsync("travelinfo.txt")!, "text/plain"), new OpenAIFileUploadExecutionSettings("travelinfo.txt", OpenAIFilePurpose.Assistants)); - Console.WriteLine(this.ApiKey); - // Define the agent OpenAIAssistantAgent agent = await OpenAIAssistantAgent.CreateAsync( diff --git a/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs b/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs index 99c14ab357a4..c20ceb662428 100644 --- a/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs +++ b/dotnet/samples/Concepts/AudioToText/OpenAI_AudioToText.cs @@ -42,7 +42,7 @@ public async Task AudioToTextAsync() // Read audio content from a file await using var audioFileStream = EmbeddedResource.ReadStream(AudioFilename); var audioFileBinaryData = await BinaryData.FromStreamAsync(audioFileStream!); - AudioContent audioContent = new(audioFileBinaryData); + AudioContent audioContent = new(audioFileBinaryData, mimeType: null); // Convert audio to text var textContent = await audioToTextService.GetTextContentAsync(audioContent, executionSettings); diff --git a/dotnet/samples/Concepts/ChatCompletion/ChatHistorySerialization.cs b/dotnet/samples/Concepts/ChatCompletion/ChatHistorySerialization.cs index c174dbe732c7..e98f1861b6d9 100644 --- a/dotnet/samples/Concepts/ChatCompletion/ChatHistorySerialization.cs +++ b/dotnet/samples/Concepts/ChatCompletion/ChatHistorySerialization.cs @@ -17,7 +17,7 @@ public class ChatHistorySerialization(ITestOutputHelper output) : BaseTest(outpu /// with having SK various content types as items. /// [Fact] - public void SerializeChatHistoryWithSKContentTypes() + public async Task SerializeChatHistoryWithSKContentTypesAsync() { int[] data = [1, 2, 3]; @@ -27,10 +27,8 @@ public void SerializeChatHistoryWithSKContentTypes() [ new TextContent("Discuss the potential long-term consequences for the Earth's ecosystem as well."), new ImageContent(new Uri("https://fake-random-test-host:123")), - new BinaryContent(new BinaryData(data)), -#pragma warning disable SKEXP0001 - new AudioContent(new BinaryData(data)) -#pragma warning restore SKEXP0001 + new BinaryContent(new BinaryData(data), "application/octet-stream"), + new AudioContent(new BinaryData(data), "application/octet-stream") ] }; @@ -49,7 +47,7 @@ public void SerializeChatHistoryWithSKContentTypes() Console.WriteLine($"Image content: {(deserializedMessage.Items![1]! as ImageContent)!.Uri}"); - Console.WriteLine($"Binary content: {Encoding.UTF8.GetString((deserializedMessage.Items![2]! as BinaryContent)!.Content!.Value.Span)}"); + Console.WriteLine($"Binary content: {Encoding.UTF8.GetString((deserializedMessage.Items![2]! as BinaryContent)!.Data!.Value.Span)}"); Console.WriteLine($"Audio content: {Encoding.UTF8.GetString((deserializedMessage.Items![3]! as AudioContent)!.Data!.Value.Span)}"); diff --git a/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs b/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs index 43c42ffc899a..1bf70ca28f5b 100644 --- a/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs +++ b/dotnet/samples/Concepts/ChatCompletion/Google_GeminiVision.cs @@ -41,7 +41,7 @@ public async Task GoogleAIAsync() new TextContent("What’s in this image?"), // Google AI Gemini API requires the image to be in base64 format, doesn't support URI // You have to always provide the mimeType for the image - new ImageContent(bytes) { MimeType = "image/jpeg" }, + new ImageContent(bytes, "image/jpeg"), ]); var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory); @@ -109,7 +109,7 @@ public async Task VertexAIAsync() new TextContent("What’s in this image?"), // Vertex AI Gemini API supports both base64 and URI format // You have to always provide the mimeType for the image - new ImageContent(bytes) { MimeType = "image/jpeg" }, + new ImageContent(bytes, "image/jpeg"), // The Cloud Storage URI of the image to include in the prompt. // The bucket that stores the file must be in the same Google Cloud project that's sending the request. // new ImageContent(new Uri("gs://generativeai-downloads/images/scones.jpg"), diff --git a/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithVision.cs b/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithVision.cs index c98830fe6e9e..2d9159448419 100644 --- a/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithVision.cs +++ b/dotnet/samples/Concepts/ChatCompletion/OpenAI_ChatCompletionWithVision.cs @@ -49,7 +49,7 @@ public async Task LocalImageAsync() chatHistory.AddUserMessage( [ new TextContent("What’s in this image?"), - new ImageContent(imageBytes) { MimeType = "image/jpg" } + new ImageContent(imageBytes, "image/jpg") ]); var reply = await chatCompletionService.GetChatMessageContentAsync(chatHistory); diff --git a/dotnet/samples/Concepts/ImageToText/HuggingFace_ImageToText.cs b/dotnet/samples/Concepts/ImageToText/HuggingFace_ImageToText.cs index 92f32e78cca1..92ab11d9c3b9 100644 --- a/dotnet/samples/Concepts/ImageToText/HuggingFace_ImageToText.cs +++ b/dotnet/samples/Concepts/ImageToText/HuggingFace_ImageToText.cs @@ -35,10 +35,7 @@ public async Task ImageToTextAsync() // Read image content from a file ReadOnlyMemory imageData = await EmbeddedResource.ReadAllAsync(ImageFilePath); - ImageContent imageContent = new(new BinaryData(imageData)) - { - MimeType = "image/jpeg" - }; + ImageContent imageContent = new(new BinaryData(imageData), "image/jpeg"); // Convert image to text var textContent = await imageToText.GetTextContentAsync(imageContent, executionSettings); diff --git a/dotnet/samples/Demos/HuggingFaceImageToText/FormMain.cs b/dotnet/samples/Demos/HuggingFaceImageToText/FormMain.cs index eeb67784603e..e269c265489b 100644 --- a/dotnet/samples/Demos/HuggingFaceImageToText/FormMain.cs +++ b/dotnet/samples/Demos/HuggingFaceImageToText/FormMain.cs @@ -143,10 +143,7 @@ private void UpdateImageDescription(string description) /// The target . /// Returns a . private static ImageContent CreateImageContentFromPictureBox(PictureBox pictureBox) - => new(ConvertImageToReadOnlyMemory(pictureBox)) - { - MimeType = GetMimeType(pictureBox.Tag?.ToString()!) - }; + => new(ConvertImageToReadOnlyMemory(pictureBox), GetMimeType(pictureBox.Tag?.ToString()!)); /// /// Gets the image binary array from a . diff --git a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs index daeac8d69f1b..4053fb8ee79f 100644 --- a/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs +++ b/dotnet/src/Connectors/Connectors.Google.UnitTests/Core/Gemini/GeminiRequestTests.cs @@ -177,7 +177,7 @@ public void FromChatHistoryImageAsImageContentItReturnsGeminiRequestWithChatHist chatHistory.AddUserMessage(contentItems: [new ImageContent(new Uri("https://example-image.com/")) { MimeType = "image/png" }]); chatHistory.AddUserMessage(contentItems: - [new ImageContent(imageAsBytes) { MimeType = "image/png" }]); + [new ImageContent(imageAsBytes, "image/png")]); var executionSettings = new GeminiPromptExecutionSettings(); // Act diff --git a/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs b/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs index 6f5b4d3d2954..2a0cf135adaa 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/AzureSdk/ClientCore.cs @@ -340,6 +340,11 @@ internal async Task> GetTextContentFromAudioAsync( CancellationToken cancellationToken) { Verify.NotNull(content.Data); + var audioData = content.Data.Value; + if (audioData.IsEmpty) + { + throw new ArgumentException("Audio data cannot be empty", nameof(content)); + } OpenAIAudioToTextExecutionSettings? audioExecutionSettings = OpenAIAudioToTextExecutionSettings.FromExecutionSettings(executionSettings); @@ -347,7 +352,7 @@ internal async Task> GetTextContentFromAudioAsync( var audioOptions = new AudioTranscriptionOptions { - AudioData = BinaryData.FromBytes(content.Data.Value), + AudioData = BinaryData.FromBytes(audioData), DeploymentName = this.DeploymentOrModelName, Filename = audioExecutionSettings.Filename, Language = audioExecutionSettings.Language, @@ -1241,13 +1246,13 @@ private static List GetRequestMessages(ChatMessageContent me if (resultContent.Result is Exception ex) { - toolMessages.Add(new ChatRequestToolMessage($"Error: Exception while invoking function. {ex.Message}", resultContent.Id)); + toolMessages.Add(new ChatRequestToolMessage($"Error: Exception while invoking function. {ex.Message}", resultContent.CallId)); continue; } var stringResult = ProcessFunctionResult(resultContent.Result ?? string.Empty, toolCallBehavior); - toolMessages.Add(new ChatRequestToolMessage(stringResult ?? string.Empty, resultContent.Id)); + toolMessages.Add(new ChatRequestToolMessage(stringResult ?? string.Empty, resultContent.CallId)); } if (toolMessages is not null) diff --git a/dotnet/src/Connectors/Connectors.OpenAI/CompatibilitySuppressions.xml b/dotnet/src/Connectors/Connectors.OpenAI/CompatibilitySuppressions.xml new file mode 100644 index 000000000000..1dd99a9223a4 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.OpenAI/CompatibilitySuppressions.xml @@ -0,0 +1,18 @@ + + + + + CP0002 + M:Microsoft.SemanticKernel.Connectors.OpenAI.OpenAIFileService.GetFileContent(System.String,System.Threading.CancellationToken) + lib/net8.0/Microsoft.SemanticKernel.Connectors.OpenAI.dll + lib/net8.0/Microsoft.SemanticKernel.Connectors.OpenAI.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.Connectors.OpenAI.OpenAIFileService.GetFileContent(System.String,System.Threading.CancellationToken) + lib/netstandard2.0/Microsoft.SemanticKernel.Connectors.OpenAI.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Connectors.OpenAI.dll + true + + \ No newline at end of file diff --git a/dotnet/src/Connectors/Connectors.OpenAI/Files/OpenAIFileService.cs b/dotnet/src/Connectors/Connectors.OpenAI/Files/OpenAIFileService.cs index 1efce6172f8d..cc61734f44c8 100644 --- a/dotnet/src/Connectors/Connectors.OpenAI/Files/OpenAIFileService.cs +++ b/dotnet/src/Connectors/Connectors.OpenAI/Files/OpenAIFileService.cs @@ -109,11 +109,22 @@ public async Task DeleteFileAsync(string id, CancellationToken cancellationToken /// /// Files uploaded with do not support content retrieval. /// - public BinaryContent GetFileContent(string id, CancellationToken cancellationToken = default) + public async Task GetFileContentAsync(string id, CancellationToken cancellationToken = default) { Verify.NotNull(id, nameof(id)); + var (stream, mimetype) = await this.StreamGetRequestAsync($"{this._serviceUri}/{id}/content", cancellationToken).ConfigureAwait(false); - return new BinaryContent(() => this.StreamGetRequestAsync($"{this._serviceUri}/{id}/content", cancellationToken)); + using (stream) + { + using var memoryStream = new MemoryStream(); +#if NETSTANDARD2_0 + const int DefaultCopyBufferSize = 81920; + await stream.CopyToAsync(memoryStream, DefaultCopyBufferSize, cancellationToken).ConfigureAwait(false); +#else + await stream.CopyToAsync(memoryStream, cancellationToken).ConfigureAwait(false); +#endif + return new BinaryContent(memoryStream.ToArray(), mimetype); + } } /// @@ -153,11 +164,11 @@ public async Task> GetFilesAsync(CancellationTo public async Task UploadContentAsync(BinaryContent fileContent, OpenAIFileUploadExecutionSettings settings, CancellationToken cancellationToken = default) { Verify.NotNull(settings, nameof(settings)); + Verify.NotNull(fileContent.Data, nameof(fileContent.Data)); using var formData = new MultipartFormDataContent(); using var contentPurpose = new StringContent(this.ConvertPurpose(settings.Purpose)); - using var contentStream = await fileContent.GetStreamAsync().ConfigureAwait(false); - using var contentFile = new StreamContent(contentStream); + using var contentFile = new ByteArrayContent(fileContent.Data.Value.ToArray()); formData.Add(contentPurpose, "purpose"); formData.Add(contentFile, "file", settings.FileName); @@ -191,7 +202,7 @@ private async Task ExecuteGetRequestAsync(string url, Cancellati }; } - private async Task StreamGetRequestAsync(string url, CancellationToken cancellationToken) + private async Task<(Stream Stream, string? MimeType)> StreamGetRequestAsync(string url, CancellationToken cancellationToken) { using var request = HttpRequest.CreateGetRequest(this.PrepareUrl(url)); this.AddRequestHeaders(request); @@ -199,9 +210,10 @@ private async Task StreamGetRequestAsync(string url, CancellationToken c try { return - new HttpResponseStream( + (new HttpResponseStream( await response.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false), - response); + response), + response.Content.Headers.ContentType?.MediaType); } catch { diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/AzureOpenAIAudioToTextServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/AzureOpenAIAudioToTextServiceTests.cs index 01690da354a8..6100c434c878 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/AzureOpenAIAudioToTextServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/AzureOpenAIAudioToTextServiceTests.cs @@ -88,7 +88,7 @@ public async Task GetTextContentWithInvalidSettingsThrowsExceptionAsync(OpenAIAu }; // Act - var exception = await Record.ExceptionAsync(() => service.GetTextContentsAsync(new AudioContent(new BinaryData("data")), settings)); + var exception = await Record.ExceptionAsync(() => service.GetTextContentsAsync(new AudioContent(new BinaryData("data"), mimeType: null), settings)); // Assert Assert.NotNull(exception); @@ -106,7 +106,7 @@ public async Task GetTextContentByDefaultWorksCorrectlyAsync() }; // Act - var result = await service.GetTextContentsAsync(new AudioContent(new BinaryData("data")), new OpenAIAudioToTextExecutionSettings("file.mp3")); + var result = await service.GetTextContentsAsync(new AudioContent(new BinaryData("data"), mimeType: null), new OpenAIAudioToTextExecutionSettings("file.mp3")); // Assert Assert.NotNull(result); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/OpenAIAudioToTextServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/OpenAIAudioToTextServiceTests.cs index c9140935798b..40959c7c67ed 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/OpenAIAudioToTextServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/AudioToText/OpenAIAudioToTextServiceTests.cs @@ -70,7 +70,7 @@ public async Task GetTextContentByDefaultWorksCorrectlyAsync() }; // Act - var result = await service.GetTextContentsAsync(new AudioContent(new BinaryData("data")), new OpenAIAudioToTextExecutionSettings("file.mp3")); + var result = await service.GetTextContentsAsync(new AudioContent(new BinaryData("data"), mimeType: null), new OpenAIAudioToTextExecutionSettings("file.mp3")); // Assert Assert.NotNull(result); diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Files/OpenAIFileServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Files/OpenAIFileServiceTests.cs index b2a3f8b7b6c2..b9619fc1bc58 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Files/OpenAIFileServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/Files/OpenAIFileServiceTests.cs @@ -196,8 +196,8 @@ public async Task GetFileContentWorksCorrectlyAsync(bool isAzure) }; // Act & Assert - var content = service.GetFileContent("file-id"); - var result = await content.GetContentAsync(); + var content = await service.GetFileContentAsync("file-id"); + var result = content.Data!.Value; Assert.Equal(data.ToArray(), result.ToArray()); } @@ -236,7 +236,7 @@ public async Task UploadContentWorksCorrectlyAsync(bool isAzure, bool isFailedRe stream.Position = 0; - var content = new BinaryContent(() => Task.FromResult(stream)); + var content = new BinaryContent(stream.ToArray(), "text/plain"); // Act & Assert if (isFailedRequest) diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/AzureOpenAITextToAudioServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/AzureOpenAITextToAudioServiceTests.cs index 518cfbaaadde..baa11a265f0a 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/AzureOpenAITextToAudioServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/AzureOpenAITextToAudioServiceTests.cs @@ -84,8 +84,9 @@ public async Task GetAudioContentByDefaultWorksCorrectlyAsync() var result = await service.GetAudioContentsAsync("Some text", new OpenAITextToAudioExecutionSettings("voice")); // Assert - Assert.NotNull(result[0].Data); - Assert.True(result[0].Data!.Value.Span.SequenceEqual(expectedByteArray)); + var audioData = result[0].Data!.Value; + Assert.False(audioData.IsEmpty); + Assert.True(audioData.Span.SequenceEqual(expectedByteArray)); } [Theory] diff --git a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/OpenAITextToAudioServiceTests.cs b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/OpenAITextToAudioServiceTests.cs index 6c3c6746f511..588616f54348 100644 --- a/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/OpenAITextToAudioServiceTests.cs +++ b/dotnet/src/Connectors/Connectors.UnitTests/OpenAI/TextToAudio/OpenAITextToAudioServiceTests.cs @@ -83,8 +83,9 @@ public async Task GetAudioContentByDefaultWorksCorrectlyAsync() var result = await service.GetAudioContentsAsync("Some text", new OpenAITextToAudioExecutionSettings("voice")); // Assert - Assert.NotNull(result[0].Data); - Assert.True(result[0].Data!.Value.Span.SequenceEqual(expectedByteArray)); + var audioData = result[0].Data!.Value; + Assert.False(audioData.IsEmpty); + Assert.True(audioData.Span.SequenceEqual(expectedByteArray)); } [Theory] diff --git a/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs b/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs index afd579c6bc45..321ede0ff115 100644 --- a/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/Google/Gemini/GeminiChatCompletionTests.cs @@ -75,7 +75,7 @@ public async Task ChatGenerationVisionBinaryDataAsync(ServiceType serviceType) var messageContent = new ChatMessageContent(AuthorRole.User, items: [ new TextContent("This is an image with a car. Which color is it? You can chose from red, blue, green, and yellow"), - new ImageContent(image) { MimeType = "image/jpeg" } + new ImageContent(image, "image/jpeg") ]); chatHistory.Add(messageContent); @@ -101,7 +101,7 @@ public async Task ChatStreamingVisionBinaryDataAsync(ServiceType serviceType) var messageContent = new ChatMessageContent(AuthorRole.User, items: [ new TextContent("This is an image with a car. Which color is it? You can chose from red, blue, green, and yellow"), - new ImageContent(image) { MimeType = "image/jpeg" } + new ImageContent(image, "image/jpeg") ]); chatHistory.Add(messageContent); diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIAudioToTextTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIAudioToTextTests.cs index 219b5d009dbe..02083e117548 100644 --- a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIAudioToTextTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIAudioToTextTests.cs @@ -40,7 +40,7 @@ public async Task OpenAIAudioToTextTestAsync() var audioData = await BinaryData.FromStreamAsync(audio); // Act - var result = await service.GetTextContentAsync(new AudioContent(audioData), new OpenAIAudioToTextExecutionSettings(Filename)); + var result = await service.GetTextContentAsync(new AudioContent(audioData, mimeType: "audio/wav"), new OpenAIAudioToTextExecutionSettings(Filename)); // Assert Assert.Contains("The sun rises in the east and sets in the west.", result.Text, StringComparison.OrdinalIgnoreCase); @@ -68,7 +68,7 @@ public async Task AzureOpenAIAudioToTextTestAsync() var audioData = await BinaryData.FromStreamAsync(audio); // Act - var result = await service.GetTextContentAsync(new AudioContent(audioData), new OpenAIAudioToTextExecutionSettings(Filename)); + var result = await service.GetTextContentAsync(new AudioContent(audioData, mimeType: "audio/wav"), new OpenAIAudioToTextExecutionSettings(Filename)); // Assert Assert.Contains("The sun rises in the east and sets in the west.", result.Text, StringComparison.OrdinalIgnoreCase); diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextToAudioTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextToAudioTests.cs index 140cf7b10fa8..e35c357cf375 100644 --- a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextToAudioTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAITextToAudioTests.cs @@ -35,8 +35,8 @@ public async Task OpenAITextToAudioTestAsync() var result = await service.GetAudioContentAsync("The sun rises in the east and sets in the west."); // Assert - Assert.NotNull(result.Data); - Assert.False(result.Data!.Value.IsEmpty); + var audioData = result.Data!.Value; + Assert.False(audioData.IsEmpty); } [Fact] @@ -59,7 +59,7 @@ public async Task AzureOpenAITextToAudioTestAsync() var result = await service.GetAudioContentAsync("The sun rises in the east and sets in the west."); // Assert - Assert.NotNull(result.Data); - Assert.False(result.Data!.Value.IsEmpty); + var audioData = result.Data!.Value; + Assert.False(audioData.IsEmpty); } } diff --git a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIToolsTests.cs b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIToolsTests.cs index 7df3c32648a9..ebfcccd31472 100644 --- a/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIToolsTests.cs +++ b/dotnet/src/IntegrationTests/Connectors/OpenAI/OpenAIToolsTests.cs @@ -436,7 +436,7 @@ public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForAutoFu var getCurrentTimeFunctionCallResult = getCurrentTimeFunctionCallResultMessage.Items.OfType().Single(); Assert.Equal("GetCurrentUtcTime", getCurrentTimeFunctionCallResult.FunctionName); Assert.Equal("HelperFunctions", getCurrentTimeFunctionCallResult.PluginName); - Assert.Equal(getCurrentTimeFunctionCallRequest.Id, getCurrentTimeFunctionCallResult.Id); + Assert.Equal(getCurrentTimeFunctionCallRequest.Id, getCurrentTimeFunctionCallResult.CallId); Assert.NotNull(getCurrentTimeFunctionCallResult.Result); // LLM requested the weather for Boston. @@ -456,7 +456,7 @@ public async Task ConnectorAgnosticFunctionCallingModelClassesCanBeUsedForAutoFu var getWeatherForCityFunctionCallResult = getWeatherForCityFunctionCallResultMessage.Items.OfType().Single(); Assert.Equal("Get_Weather_For_City", getWeatherForCityFunctionCallResult.FunctionName); Assert.Equal("HelperFunctions", getWeatherForCityFunctionCallResult.PluginName); - Assert.Equal(getWeatherForCityFunctionCallRequest.Id, getWeatherForCityFunctionCallResult.Id); + Assert.Equal(getWeatherForCityFunctionCallRequest.Id, getWeatherForCityFunctionCallResult.CallId); Assert.NotNull(getWeatherForCityFunctionCallResult.Result); } diff --git a/dotnet/src/InternalUtilities/samples/InternalUtilities/BaseTest.cs b/dotnet/src/InternalUtilities/samples/InternalUtilities/BaseTest.cs index 1848734b6218..dadf49c15d27 100644 --- a/dotnet/src/InternalUtilities/samples/InternalUtilities/BaseTest.cs +++ b/dotnet/src/InternalUtilities/samples/InternalUtilities/BaseTest.cs @@ -76,9 +76,7 @@ protected BaseTest(ITestOutputHelper output) /// /// Target object to write public void WriteLine(object? target = null) - { - this.Output.WriteLine(target ?? string.Empty); - } + => this.Output.WriteLine(target ?? string.Empty); /// /// This method can be substituted by Console.WriteLine when used in Console apps. @@ -88,6 +86,12 @@ public void WriteLine(object? target = null) public void WriteLine(string? format, params object?[] args) => this.Output.WriteLine(format ?? string.Empty, args); + /// + /// This method can be substituted by Console.WriteLine when used in Console apps. + /// + public void WriteLine(string? message) + => this.Output.WriteLine(message); + /// /// Current interface ITestOutputHelper does not have a Write method. This extension method adds it to make it analogous to Console.Write when used in Console apps. /// diff --git a/dotnet/src/InternalUtilities/src/Text/DataUriParser.cs b/dotnet/src/InternalUtilities/src/Text/DataUriParser.cs new file mode 100644 index 000000000000..41887a4cbe79 --- /dev/null +++ b/dotnet/src/InternalUtilities/src/Text/DataUriParser.cs @@ -0,0 +1,172 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; + +#pragma warning disable CA1307 // Specify StringComparison +#pragma warning disable CA1847 // Use StringBuilder.Append when concatenating strings + +namespace Microsoft.SemanticKernel.Text; + +/// +/// Data Uri Scheme Parser based on RFC 2397. +/// https://datatracker.ietf.org/doc/html/rfc2397 +/// +[ExcludeFromCodeCoverage] +internal static class DataUriParser +{ + private const string Scheme = "data:"; + + private static readonly char[] s_base64Chars = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' + }; + /// + /// Extension method to test whether the value is a base64 string + /// + /// Value to test + /// Boolean value, true if the string is base64, otherwise false + private static bool IsBase64String(string? value) + { + // The quickest test. If the value is null or is equal to 0 it is not base64 + // Base64 string's length is always divisible by four, i.e. 8, 16, 20 etc. + // If it is not you can return false. Quite effective + // Further, if it meets the above criteria, then test for spaces. + // If it contains spaces, it is not base64 + if (value is null + || value.Length == 0 + || value.Length % 4 != 0 + || value.Contains(' ') + || value.Contains('\t') + || value.Contains('\r') + || value.Contains('\n')) + { + return false; + } + + // 98% of all non base64 values are invalidated by this time. + var index = value.Length - 1; + + // if there is padding step back + if (value[index] == '=') { index--; } + + // if there are two padding chars step back a second time + if (value[index] == '=') { index--; } + + // Now traverse over characters + for (var i = 0; i <= index; i++) + { + // If any of the character is not from the allowed list + if (!s_base64Chars.Contains(value[i])) + { + // return false + return false; + } + } + + // If we got here, then the value is a valid base64 string + return true; + } + + internal static DataUri Parse(string? dataUri) + { + Verify.NotNullOrWhiteSpace(dataUri, nameof(dataUri)); + if (!dataUri.StartsWith(Scheme, StringComparison.OrdinalIgnoreCase)) + { + throw new UriFormatException("Invalid data uri format. The data URI must start with 'data:'."); + } + + var model = new DataUri(); + int currentIndex = Scheme.Length; + int dataIndex = dataUri.IndexOf(',', currentIndex); + + if (dataIndex == -1) + { + throw new UriFormatException("Invalid data uri format. The data URI must contain a comma separating the metadata and the data."); + } + + string metadata = dataUri.Substring(currentIndex, dataIndex - currentIndex); + model.Data = dataUri.Substring(dataIndex + 1); + + // Split the metadata into components + var metadataParts = metadata.Split(';'); + if (metadataParts.Length > 0) + { + if (!string.IsNullOrWhiteSpace(metadataParts[0]) && !metadataParts[0].Contains("/")) + { + throw new UriFormatException("Invalid data uri format. When provided, the MIME type must have \"type/subtype\" format."); + } + + // First part is the MIME type + model.MimeType = metadataParts[0]; + } + + for (int i = 1; i < metadataParts.Length; i++) + { + var part = metadataParts[i]; + if (part!.Contains("=")) + { + var keyValue = part.Split('='); + + // Parameter must have a name and cannot have more than one '=' for values. + if (string.IsNullOrWhiteSpace(keyValue[0]) || keyValue.Length != 2) + { + throw new UriFormatException("Invalid data uri format. Parameters must have \"name=value\" format."); + } + + model.Parameters[keyValue[0]] = keyValue[1]; + + continue; + } + + if (i < metadataParts.Length - 1) + { + throw new UriFormatException("Invalid data uri format. Parameters must have \"name=value\" format."); + } + + model.DataFormat = part; + } + + if (string.Equals(model.DataFormat, "base64", StringComparison.OrdinalIgnoreCase) && !IsBase64String(model.Data)) + { + throw new UriFormatException("Invalid data uri format. The data is not a valid Base64 string."); + } + + if (string.IsNullOrEmpty(model.MimeType)) + { + // By RFC 2397, the default MIME type if not provided is text/plain;charset=US-ASCII + model.MimeType = "text/plain"; + } + + return model; + } + + /// + /// Represents the data URI parts. + /// + internal sealed class DataUri + { + /// + /// The mime type of the data. + /// + internal string? MimeType { get; set; } + + /// + /// The optional parameters of the data. + /// + internal Dictionary Parameters { get; set; } = new(); + + /// + /// The optional format of the data. Most common is "base64". + /// + public string? DataFormat { get; set; } + + /// + /// The data content. + /// + public string? Data { get; set; } + } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/CompatibilitySuppressions.xml b/dotnet/src/SemanticKernel.Abstractions/CompatibilitySuppressions.xml new file mode 100644 index 000000000000..c000a4e54c9b --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/CompatibilitySuppressions.xml @@ -0,0 +1,130 @@ + + + + + CP0002 + M:Microsoft.SemanticKernel.AudioContent.#ctor(System.ReadOnlyMemory{System.Byte},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.#ctor(System.Func{System.Threading.Tasks.Task{System.IO.Stream}},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.#ctor(System.ReadOnlyMemory{System.Byte},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.GetContentAsync + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.GetStreamAsync + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.set_Content(System.Nullable{System.ReadOnlyMemory{System.Byte}}) + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.FunctionResultContent.get_Id + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.ImageContent.#ctor(System.ReadOnlyMemory{System.Byte},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.ImageContent.#ctor(System.Uri,System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + lib/net8.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.AudioContent.#ctor(System.ReadOnlyMemory{System.Byte},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.#ctor(System.Func{System.Threading.Tasks.Task{System.IO.Stream}},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.#ctor(System.ReadOnlyMemory{System.Byte},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.GetContentAsync + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.GetStreamAsync + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.BinaryContent.set_Content(System.Nullable{System.ReadOnlyMemory{System.Byte}}) + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.FunctionResultContent.get_Id + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.ImageContent.#ctor(System.ReadOnlyMemory{System.Byte},System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + + CP0002 + M:Microsoft.SemanticKernel.ImageContent.#ctor(System.Uri,System.String,System.Object,System.Collections.Generic.IReadOnlyDictionary{System.String,System.Object}) + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + lib/netstandard2.0/Microsoft.SemanticKernel.Abstractions.dll + true + + \ No newline at end of file diff --git a/dotnet/src/SemanticKernel.Abstractions/Contents/AudioContent.cs b/dotnet/src/SemanticKernel.Abstractions/Contents/AudioContent.cs index e6b894b048b7..588d51924de6 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Contents/AudioContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Contents/AudioContent.cs @@ -1,45 +1,49 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Text.Json.Serialization; +#pragma warning disable CA1054 // URI-like parameters should not be strings + namespace Microsoft.SemanticKernel; /// /// Represents audio content. /// [Experimental("SKEXP0001")] -public class AudioContent : KernelContent +public class AudioContent : BinaryContent { /// - /// The audio data. + /// Initializes a new instance of the class. /// - public ReadOnlyMemory? Data { get; set; } + [JsonConstructor] + public AudioContent() + { + } /// /// Initializes a new instance of the class. /// - [JsonConstructor] - public AudioContent() + /// The URI of audio. + public AudioContent(Uri uri) : base(uri) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// DataUri of the audio + public AudioContent(string dataUri) : base(dataUri) { } /// /// Initializes a new instance of the class. /// - /// The audio binary data. - /// The model ID used to generate the content. - /// Inner content, - /// Additional metadata - public AudioContent( - ReadOnlyMemory data, - string? modelId = null, - object? innerContent = null, - IReadOnlyDictionary? metadata = null) - : base(innerContent, modelId, metadata) + /// Byte array of the audio + /// Mime type of the audio + public AudioContent(ReadOnlyMemory data, string? mimeType) : base(data, mimeType) { - this.Data = data; } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Contents/BinaryContent.cs b/dotnet/src/SemanticKernel.Abstractions/Contents/BinaryContent.cs index 2bb2a904bc7d..d7424eca571a 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Contents/BinaryContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Contents/BinaryContent.cs @@ -3,118 +3,315 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.IO; +using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; +using Microsoft.SemanticKernel.Text; + +#pragma warning disable CA1056 // URI-like properties should not be strings +#pragma warning disable CA1055 // URI-like parameters should not be strings +#pragma warning disable CA1054 // URI-like parameters should not be strings namespace Microsoft.SemanticKernel; /// /// Provides access to binary content. /// -[Experimental("SKEXP0010")] +[Experimental("SKEXP0001")] public class BinaryContent : KernelContent { - private readonly Func>? _streamProvider; + private string? _dataUri; + private ReadOnlyMemory? _data; + private Uri? _referencedUri; /// /// The binary content. /// - public ReadOnlyMemory? Content { get; set; } + [JsonIgnore, Obsolete("Use Data instead")] + public ReadOnlyMemory? Content => this.Data; /// - /// Initializes a new instance of the class. + /// Gets the referenced Uri of the content. /// - [JsonConstructor] - public BinaryContent() + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public Uri? Uri { + get => this.GetUri(); + set => this.SetUri(value); } /// - /// Initializes a new instance of the class. + /// Gets the DataUri of the content. /// - /// The binary content - /// The model ID used to generate the content - /// Inner content - /// Additional metadata - public BinaryContent( - ReadOnlyMemory content, - string? modelId = null, - object? innerContent = null, - IReadOnlyDictionary? metadata = null) - : base(innerContent, modelId, metadata) + [JsonIgnore] + public string? DataUri { - Verify.NotNull(content, nameof(content)); + get => this.GetDataUri(); + set => this.SetDataUri(value); + } - this.Content = content; + /// + /// Gets the byte array data of the content. + /// + [JsonPropertyOrder(100), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] // Ensuring Data Uri is serialized last for better visibility of other properties. + public ReadOnlyMemory? Data + { + get => this.GetData(); + set => this.SetData(value); } /// - /// Initializes a new instance of the class. + /// Indicates whether the content contains binary data in either or properties. + /// + /// True if the content has binary data, false otherwise. + [JsonIgnore] + public bool CanRead + => this._data is not null + || this._dataUri is not null; + + /// + /// Initializes a new instance of the class with no content. /// - /// The asynchronous stream provider. - /// The model ID used to generate the content - /// Inner content - /// Additional metadata /// - /// The is accessed and disposed as part of either the - /// the or - /// accessor methods. + /// Should be used only for serialization purposes. /// + [JsonConstructor] + public BinaryContent() + { + } + + /// + /// Initializes a new instance of the class referring to an external uri. + /// + public BinaryContent(Uri uri) + { + this.Uri = uri; + } + + /// + /// Initializes a new instance of the class for a UriData or Uri referred content. + /// + /// The Uri of the content. public BinaryContent( - Func> streamProvider, - string? modelId = null, - object? innerContent = null, - IReadOnlyDictionary? metadata = null) - : base(innerContent, modelId, metadata) + // Uri type has a ushort size limit check which inviabilizes its usage in DataUri scenarios. + // + string dataUri) { - Verify.NotNull(streamProvider, nameof(streamProvider)); + this.DataUri = dataUri; + } - this._streamProvider = streamProvider; + /// + /// Initializes a new instance of the class from a byte array. + /// + /// Byte array content + /// The mime type of the content + public BinaryContent( + ReadOnlyMemory data, + string? mimeType) + { + Verify.NotNull(data, nameof(data)); + if (data.IsEmpty) + { + throw new ArgumentException("Data cannot be empty", nameof(data)); + } + + this.MimeType = mimeType; + this.Data = data; } + #region Private /// - /// Access the content stream. + /// Sets the Uri of the content. /// - /// - /// Caller responsible for disposal. - /// - public async Task GetStreamAsync() + /// Target Uri + private void SetUri(Uri? uri) + { + if (uri?.Scheme == "data") + { + throw new InvalidOperationException("For DataUri contents, use DataUri property."); + } + + this._referencedUri = uri; + } + + /// + /// Gets the Uri of the content. + /// + /// Uri of the content + private Uri? GetUri() + => this._referencedUri; + + /// + /// Sets the DataUri of the content. + /// + /// DataUri of the content + private void SetDataUri(string? dataUri) + { + if (dataUri is null) + { + this._dataUri = null; + + // Invalidate the current bytearray + this._data = null; + return; + } + + var isDataUri = dataUri!.StartsWith("data:", StringComparison.OrdinalIgnoreCase) == true; + if (!isDataUri) + { + throw new UriFormatException("Invalid data uri. Scheme should start with \"data:\""); + } + + // Validate the dataUri format + var parsedDataUri = DataUriParser.Parse(dataUri); + + // Overwrite the mimetype to the DataUri. + this.MimeType = parsedDataUri.MimeType; + + // If parameters where provided in the data uri, updates the content metadata. + if (parsedDataUri.Parameters.Count != 0) + { + // According to the RFC 2397, the data uri supports custom parameters + // This method ensures that if parameter is provided those will be added + // to the content metadata with a "data-uri-" prefix. + this.UpdateDataUriParametersToMetadata(parsedDataUri); + } + + this._dataUri = dataUri; + + // Invalidate the current bytearray + this._data = null; + } + + private void UpdateDataUriParametersToMetadata(DataUriParser.DataUri parsedDataUri) { - if (this._streamProvider is not null) + if (parsedDataUri.Parameters.Count == 0) + { + return; + } + + var newMetadata = this.Metadata as Dictionary; + if (newMetadata is null) { - return await this._streamProvider.Invoke().ConfigureAwait(false); + newMetadata = new Dictionary(); + if (this.Metadata is not null) + { + foreach (var property in this.Metadata!) + { + newMetadata[property.Key] = property.Value; + } + } } - if (this.Content is not null) + // Overwrite any properties if already defined + foreach (var property in parsedDataUri.Parameters) { - return new MemoryStream(this.Content.Value.ToArray()); + // Set the properties from the DataUri metadata + newMetadata[$"data-uri-{property.Key}"] = property.Value; } - throw new KernelException("Null content"); + this.Metadata = newMetadata; + } + + private string GetDataUriParametersFromMetadata() + { + var metadata = this.Metadata; + if (metadata is null || metadata.Count == 0) + { + return string.Empty; + } + + StringBuilder parameters = new(); + foreach (var property in metadata) + { + if (property.Key.StartsWith("data-uri-", StringComparison.OrdinalIgnoreCase)) + { + parameters.Append($";{property.Key.AsSpan(9).ToString()}={property.Value}"); + } + } + + return parameters.ToString(); + } + + /// + /// Sets the byte array data content. + /// + /// Byte array data content + private void SetData(ReadOnlyMemory? data) + { + // Overriding the content will invalidate the previous dataUri + this._dataUri = null; + this._data = data; } /// - /// The content stream + /// Gets the byte array data content. /// - public async Task> GetContentAsync() + /// The byte array data content + private ReadOnlyMemory? GetData() { - if (this._streamProvider is not null) + return this.GetCachedByteArrayContent(); + } + + /// + /// Gets the DataUri of the content. + /// + /// + private string? GetDataUri() + { + if (!this.CanRead) { - using var stream = await this._streamProvider.Invoke().ConfigureAwait(false); + return null; + } - using var memoryStream = new MemoryStream(); + if (this._dataUri is not null) + { + // Double check if the set MimeType matches the current dataUri. + var parsedDataUri = DataUriParser.Parse(this._dataUri); + if (string.Equals(parsedDataUri.MimeType, this.MimeType, StringComparison.OrdinalIgnoreCase)) + { + return this._dataUri; + } + } - await stream.CopyToAsync(memoryStream).ConfigureAwait(false); + // If the Uri is not a DataUri, then we need to get from byteArray (caching if needed) to generate it. + return this.GetCachedUriDataFromByteArray(this.GetCachedByteArrayContent()); + } - return memoryStream.ToArray(); + private string? GetCachedUriDataFromByteArray(ReadOnlyMemory? cachedByteArray) + { + if (cachedByteArray is null) + { + return null; } - if (this.Content is not null) + if (this.MimeType is null) + { + // May consider defaulting to application/octet-stream if not provided. + throw new InvalidOperationException("Can't get the data uri without a mime type defined for the content."); + } + + // Ensure that if any data-uri-parameter defined in the metadata those will be added to the data uri. + this._dataUri = $"data:{this.MimeType}{this.GetDataUriParametersFromMetadata()};base64," + Convert.ToBase64String(cachedByteArray.Value.ToArray()); + return this._dataUri; + } + + private ReadOnlyMemory? GetCachedByteArrayContent() + { + if (this._data is null && this._dataUri is not null) { - return this.Content.Value; + var parsedDataUri = DataUriParser.Parse(this._dataUri); + if (string.Equals(parsedDataUri.DataFormat, "base64", StringComparison.OrdinalIgnoreCase)) + { + this._data = Convert.FromBase64String(parsedDataUri.Data!); + } + else + { + // Defaults to UTF8 encoding if format is not provided. + this._data = Encoding.UTF8.GetBytes(parsedDataUri.Data!); + } } - throw new KernelException("Null content"); + return this._data; } + #endregion } diff --git a/dotnet/src/SemanticKernel.Abstractions/Contents/FunctionResultContent.cs b/dotnet/src/SemanticKernel.Abstractions/Contents/FunctionResultContent.cs index 859682d63ec1..ab1e342f7906 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Contents/FunctionResultContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Contents/FunctionResultContent.cs @@ -16,7 +16,7 @@ public sealed class FunctionResultContent : KernelContent /// The function call ID. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Id { get; } + public string? CallId { get; } /// /// The plugin name. @@ -41,14 +41,14 @@ public sealed class FunctionResultContent : KernelContent /// /// The function name. /// The plugin name. - /// The function call ID. + /// The function call ID. /// The function result. [JsonConstructor] - public FunctionResultContent(string? functionName = null, string? pluginName = null, string? id = null, object? result = null) + public FunctionResultContent(string? functionName = null, string? pluginName = null, string? callId = null, object? result = null) { this.FunctionName = functionName; this.PluginName = pluginName; - this.Id = id; + this.CallId = callId; this.Result = result; } @@ -59,7 +59,7 @@ public FunctionResultContent(string? functionName = null, string? pluginName = n /// The function result. public FunctionResultContent(FunctionCallContent functionCall, object? result = null) { - this.Id = functionCall.Id; + this.CallId = functionCall.Id; this.PluginName = functionCall.PluginName; this.FunctionName = functionCall.FunctionName; this.Result = result; diff --git a/dotnet/src/SemanticKernel.Abstractions/Contents/ImageContent.cs b/dotnet/src/SemanticKernel.Abstractions/Contents/ImageContent.cs index 2018f0653574..d555112ebbcb 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Contents/ImageContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Contents/ImageContent.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Text.Json.Serialization; namespace Microsoft.SemanticKernel; @@ -9,19 +8,8 @@ namespace Microsoft.SemanticKernel; /// /// Represents image content. /// -public sealed class ImageContent : KernelContent +public class ImageContent : BinaryContent { - /// - /// The URI of image. - /// - public Uri? Uri { get; set; } - - /// - /// The image data. - /// - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public ReadOnlyMemory? Data { get; set; } - /// /// Initializes a new instance of the class. /// @@ -34,61 +22,24 @@ public ImageContent() /// Initializes a new instance of the class. /// /// The URI of image. - /// The model ID used to generate the content - /// Inner content - /// Additional metadata - public ImageContent( - Uri uri, - string? modelId = null, - object? innerContent = null, - IReadOnlyDictionary? metadata = null) - : base(innerContent, modelId, metadata) + public ImageContent(Uri uri) : base(uri) { - this.Uri = uri; } /// /// Initializes a new instance of the class. /// - /// The Data used as DataUri for the image. - /// The model ID used to generate the content - /// Inner content - /// Additional metadata - public ImageContent( - ReadOnlyMemory data, - string? modelId = null, - object? innerContent = null, - IReadOnlyDictionary? metadata = null) - : base(innerContent, modelId, metadata) + /// DataUri of the image + public ImageContent(string dataUri) : base(dataUri) { - if (data!.IsEmpty) - { - throw new ArgumentException("Data cannot be empty", nameof(data)); - } - - this.Data = data; } /// - /// Returns the string representation of the image. - /// In-memory images will be represented as DataUri - /// Remote Uri images will be represented as is + /// Initializes a new instance of the class. /// - /// - /// When Data is provided it takes precedence over URI - /// - public override string ToString() - { - return this.BuildDataUri() ?? this.Uri?.ToString() ?? string.Empty; - } - - private string? BuildDataUri() + /// Byte array of the image + /// Mime type of the image + public ImageContent(ReadOnlyMemory data, string? mimeType) : base(data, mimeType) { - if (this.Data is null || string.IsNullOrEmpty(this.MimeType)) - { - return null; - } - - return $"data:{this.MimeType};base64,{Convert.ToBase64String(this.Data.Value.ToArray())}"; } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Contents/KernelContent.cs b/dotnet/src/SemanticKernel.Abstractions/Contents/KernelContent.cs index db9760d4db3d..183542021705 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Contents/KernelContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Contents/KernelContent.cs @@ -12,20 +12,22 @@ namespace Microsoft.SemanticKernel; [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] [JsonDerivedType(typeof(TextContent), typeDiscriminator: nameof(TextContent))] [JsonDerivedType(typeof(ImageContent), typeDiscriminator: nameof(ImageContent))] -#pragma warning disable SKEXP0010 -[JsonDerivedType(typeof(BinaryContent), typeDiscriminator: nameof(BinaryContent))] -#pragma warning restore SKEXP0010 -#pragma warning disable SKEXP0001 -[JsonDerivedType(typeof(AudioContent), typeDiscriminator: nameof(AudioContent))] [JsonDerivedType(typeof(FunctionCallContent), typeDiscriminator: nameof(FunctionCallContent))] [JsonDerivedType(typeof(FunctionResultContent), typeDiscriminator: nameof(FunctionResultContent))] -#pragma warning restore SKEXP0001 +[JsonDerivedType(typeof(BinaryContent), typeDiscriminator: nameof(BinaryContent))] +[JsonDerivedType(typeof(AudioContent), typeDiscriminator: nameof(AudioContent))] #pragma warning disable SKEXP0110 [JsonDerivedType(typeof(AnnotationContent), typeDiscriminator: nameof(AnnotationContent))] [JsonDerivedType(typeof(FileReferenceContent), typeDiscriminator: nameof(FileReferenceContent))] #pragma warning disable SKEXP0110 public abstract class KernelContent { + /// + /// MIME type of the content. + /// + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? MimeType { get; set; } + /// /// The inner content representation. Use this to bypass the current abstraction. /// @@ -47,12 +49,6 @@ public abstract class KernelContent [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IReadOnlyDictionary? Metadata { get; set; } - /// - /// MIME type of the content. - /// - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? MimeType { get; set; } - /// /// Initializes a new instance of the class. /// diff --git a/dotnet/src/SemanticKernel.UnitTests/Contents/AudioContentTests.cs b/dotnet/src/SemanticKernel.UnitTests/Contents/AudioContentTests.cs new file mode 100644 index 000000000000..c468e02b6809 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Contents/AudioContentTests.cs @@ -0,0 +1,276 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text; +using System.Text.Json; +using Microsoft.SemanticKernel; +using Xunit; + +namespace SemanticKernel.UnitTests.Contents; + +/// +/// Unit tests for class. +/// +public sealed class AudioContentTests +{ + [Fact] + public void ToStringForUriReturnsString() + { + // Arrange + var content1 = new AudioContent((Uri)null!); + var content2 = new AudioContent(new Uri("https://endpoint/")); + + // Act + var result1 = content1.ToString(); + var result2 = content2.ToString(); + + // Assert + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", result1); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", result2); + } + + [Fact] + public void ToStringForDataUriReturnsTypeString() + { + // Arrange + var data = BinaryData.FromString("this is a test"); + var content1 = new AudioContent(data, "text/plain"); + + // Act + var result1 = content1.ToString(); + var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; + + // Assert + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", result1); + } + + [Fact] + public void ToStringForUriAndDataUriReturnsDataUriString() + { + // Arrange + var data = BinaryData.FromString("this is a test"); + var content1 = new AudioContent(data, "text/plain") { Uri = new Uri("https://endpoint/") }; + + // Act + var result1 = content1.ToString(); + var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; + + // Assert + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", result1); + } + + [Fact] + public void CreateForEmptyDataUriThrows() + { + // Arrange + var data = BinaryData.Empty; + + // Assert + Assert.Throws(() + => new AudioContent(data, "text/plain")); + } + + [Fact] + public void ToStringForDataUriFromBytesReturnsType() + { + // Arrange + var bytes = System.Text.Encoding.UTF8.GetBytes("this is a test"); + var data = BinaryData.FromBytes(bytes); + var content1 = new AudioContent(data, "text/plain"); + + // Act + var result1 = content1.ToString(); + var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; + + // Assert + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", result1); + } + + [Fact] + public void ToStringForDataUriFromStreamReturnsDataUriString() + { + // Arrange + using var ms = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes("this is a test")); + var data = BinaryData.FromStream(ms); + var content1 = new AudioContent(data, "text/plain"); + + // Act + var result1 = content1.ToString(); + var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; + + // Assert + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", result1); + } + + [Fact] + public void DataConstructorWhenDataIsEmptyShouldThrow() + { + // Arrange + using var ms = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes("this is a test")); + + var data = BinaryData.FromStream(ms); + + // Assert throws if mediatype is null + Assert.Throws(() => new AudioContent(BinaryData.FromStream(ms), mimeType: null)); + } + + [Fact] + public void ToStringInMemoryImageWithoutMediaTypeReturnsType() + { + // Arrange + var sut = new AudioContent(new byte[] { 1, 2, 3 }, mimeType: null); + + // Act + var dataUrl = sut.ToString(); + + // Assert + Assert.Equal($"Microsoft.SemanticKernel.{nameof(AudioContent)}", dataUrl?.ToString()); + } + + // Ensure retrocompatibility with AudioContent Pre-BinaryContent Version + + [Theory] + [InlineData("", null, $"Microsoft.SemanticKernel.{nameof(AudioContent)}")] + [InlineData(null, null, $"Microsoft.SemanticKernel.{nameof(AudioContent)}")] + [InlineData("", "http://localhost:9090/", $"Microsoft.SemanticKernel.{nameof(AudioContent)}")] + [InlineData(null, "http://localhost:9090/", $"Microsoft.SemanticKernel.{nameof(AudioContent)}")] + [InlineData("image/png", null, $"Microsoft.SemanticKernel.{nameof(AudioContent)}")] + [InlineData("image/png", "http://localhost:9090", $"Microsoft.SemanticKernel.{nameof(AudioContent)}")] + public void ToStringShouldReturn(string? mimeType, string? path, string expectedToString) + { + // Arrange + var bytes = Encoding.UTF8.GetBytes("this is a test"); + var data = BinaryData.FromBytes(bytes); + var content1 = new AudioContent(data, mimeType); + if (path is not null) + { + content1.Uri = new Uri(path); + } + + // Act + var result1 = content1.ToString(); + + // Assert + Assert.Equal(expectedToString, result1); + } + + [Fact] + public void UpdatingUriPropertyShouldReturnAsExpected() + { + // Arrange + var data = BinaryData.FromString("this is a test"); + var content = new AudioContent(data, "text/plain"); + + // Act + var serializeBefore = JsonSerializer.Serialize(content); + + // Changing the Uri to a absolute file /foo.txt path + content.Uri = new Uri("file:///foo.txt"); + content.MimeType = "image/jpeg"; + + var serializeAfter = JsonSerializer.Serialize(content); + + // Assert + Assert.Equal("""{"MimeType":"text/plain","Data":"dGhpcyBpcyBhIHRlc3Q="}""", serializeBefore); + Assert.Equal("""{"Uri":"file:///foo.txt","MimeType":"image/jpeg","Data":"dGhpcyBpcyBhIHRlc3Q="}""", serializeAfter); + + // Uri behaves independently of other properties + Assert.Equal("file:///foo.txt", content.Uri?.ToString()); + + // Data and MimeType remain the same + Assert.Equal(Convert.FromBase64String("dGhpcyBpcyBhIHRlc3Q="), content.Data!.Value.ToArray()); + Assert.Equal(data.ToArray(), content.Data!.Value.ToArray()); + } + + [Fact] + public void UpdatingMimeTypePropertyShouldReturnAsExpected() + { + // Arrange + var data = BinaryData.FromString("this is a test"); + var content = new AudioContent(data, "text/plain"); + + // Act + var toStringBefore = content.ToString(); + + // Changing the mimetype to image/jpeg in the DataUri + Assert.Equal("data:text/plain;base64,dGhpcyBpcyBhIHRlc3Q=", content.DataUri); + + content.MimeType = "application/json"; + Assert.Equal("data:application/json;base64,dGhpcyBpcyBhIHRlc3Q=", content.DataUri); + Assert.Null(content.Uri); // Uri behaves independently of other properties, was not set, keeps null. + Assert.Equal(Convert.FromBase64String("dGhpcyBpcyBhIHRlc3Q="), content.Data!.Value.ToArray()); + Assert.Equal(data.ToArray(), content.Data!.Value.ToArray()); + Assert.Equal("application/json", content.MimeType); + } + + [Fact] + public void UpdateDataPropertyShouldReturnAsExpected() + { + // Arrange + var dataUriBefore = "data:text/plain;base64,dGhpcyBpcyBhIHRlc3Q="; + var content = new AudioContent(dataUriBefore); + + // Act + var newData = BinaryData.FromString("this is a new test"); + dataUriBefore = content.DataUri!; + content.Data = newData; + + // Assert + Assert.Equal("data:text/plain;base64,dGhpcyBpcyBhIHRlc3Q=", dataUriBefore); + Assert.Equal("data:text/plain;base64,dGhpcyBpcyBhIG5ldyB0ZXN0", content.DataUri); + Assert.Null(content.Uri); // Uri behaves independently of other properties, was not set, keeps null. + Assert.Equal("text/plain", content.MimeType); // MimeType remain the same as it was not set + Assert.Equal(Convert.FromBase64String("dGhpcyBpcyBhIG5ldyB0ZXN0"), content.Data!.Value.ToArray()); // Data is updated + } + + [Fact] + public void EmptyConstructorSerializationAndDeserializationAsExpected() + { + var content = new AudioContent(); + var serialized = JsonSerializer.Serialize(content); + var deserialized = JsonSerializer.Deserialize(serialized); + + Assert.Equal("{}", serialized); + + Assert.NotNull(deserialized); + Assert.Null(deserialized.Uri); + Assert.Null(deserialized.Data); + Assert.Null(deserialized.MimeType); + Assert.Null(deserialized.InnerContent); + Assert.Null(deserialized.ModelId); + Assert.Null(deserialized.Metadata); + } + + [Theory] + [InlineData("http://localhost:9090/")] + [InlineData(null)] + public void UriConstructorSerializationAndDeserializationAsExpected(string? path) + { +#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type. +#pragma warning disable CS8604 // Possible null reference argument. + Uri uri = path is not null ? new Uri(path) : null; + + var content = new AudioContent(uri); + var serialized = JsonSerializer.Serialize(content); + var deserialized = JsonSerializer.Deserialize(serialized); + + if (uri is null) + { + Assert.Equal("{}", serialized); + } + else + { + Assert.Equal($"{{\"Uri\":\"{uri}\"}}", serialized); + } + + Assert.NotNull(deserialized); + Assert.Equal(uri, deserialized.Uri); + Assert.Null(deserialized.Data); + Assert.Null(deserialized.MimeType); + Assert.Null(deserialized.InnerContent); + Assert.Null(deserialized.ModelId); + Assert.Null(deserialized.Metadata); +#pragma warning restore CS8604 // Possible null reference argument. +#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type. + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Contents/BinaryContentTests.cs b/dotnet/src/SemanticKernel.UnitTests/Contents/BinaryContentTests.cs new file mode 100644 index 000000000000..9ce8e356f2c4 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Contents/BinaryContentTests.cs @@ -0,0 +1,314 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Text.Json; +using Microsoft.SemanticKernel; +using Xunit; +using Xunit.Abstractions; + +namespace SemanticKernel.UnitTests.Contents; + +public sealed class BinaryContentTests(ITestOutputHelper output) +{ + [Fact] + public void ItCanBeSerialized() + { + Assert.Equal("{}", + JsonSerializer.Serialize(new BinaryContent())); + + Assert.Equal("""{"MimeType":"text/plain","Data":""}""", + JsonSerializer.Serialize(new BinaryContent("data:,"))); + + Assert.Equal("""{"Uri":"http://localhost/"}""", + JsonSerializer.Serialize(new BinaryContent(new Uri("http://localhost/")))); + + Assert.Equal("""{"MimeType":"application/octet-stream","Data":"AQIDBA=="}""", + JsonSerializer.Serialize(new BinaryContent( + dataUri: "data:application/octet-stream;base64,AQIDBA=="))); + + Assert.Equal("""{"MimeType":"application/octet-stream","Data":"AQIDBA=="}""", + JsonSerializer.Serialize(new BinaryContent( + data: new ReadOnlyMemory([0x01, 0x02, 0x03, 0x04]), + mimeType: "application/octet-stream") + )); + } + + [Fact] + public void ItCanBeDeserialized() + { + // Empty Binary Object + var content = JsonSerializer.Deserialize("{}")!; + Assert.Null(content.Data); + Assert.Null(content.DataUri); + Assert.Null(content.MimeType); + Assert.Null(content.Uri); + Assert.False(content.CanRead); + + // Data + MimeType only + content = JsonSerializer.Deserialize("""{"MimeType":"application/octet-stream","Data":"AQIDBA=="}""")!; + + Assert.Null(content.Uri); + Assert.NotNull(content.Data); + Assert.Equal(new ReadOnlyMemory([0x01, 0x02, 0x03, 0x04]), content.Data!.Value); + Assert.Equal("application/octet-stream", content.MimeType); + Assert.True(content.CanRead); + + // Uri referenced content-only + content = JsonSerializer.Deserialize("""{"MimeType":"application/octet-stream","Uri":"http://localhost/"}""")!; + + Assert.Null(content.Data); + Assert.Null(content.DataUri); + Assert.Equal("application/octet-stream", content.MimeType); + Assert.Equal(new Uri("http://localhost/"), content.Uri); + Assert.False(content.CanRead); + + // Using extra metadata + content = JsonSerializer.Deserialize(""" + { + "Data": "AQIDBA==", + "ModelId": "gpt-4", + "Metadata": { + "key": "value" + }, + "Uri": "http://localhost/myfile.txt", + "MimeType": "text/plain" + } + """)!; + + Assert.Equal(new Uri("http://localhost/myfile.txt"), content.Uri); + Assert.NotNull(content.Data); + Assert.Equal(new ReadOnlyMemory([0x01, 0x02, 0x03, 0x04]), content.Data!.Value); + Assert.Equal("text/plain", content.MimeType); + Assert.True(content.CanRead); + Assert.Equal("gpt-4", content.ModelId); + Assert.Equal("value", content.Metadata!["key"]!.ToString()); + } + + [Fact] + public void ItCanBecomeReadableAfterProvidingDataUri() + { + var content = new BinaryContent(new Uri("http://localhost/")); + Assert.False(content.CanRead); + Assert.Equal("http://localhost/", content.Uri?.ToString()); + Assert.Null(content.MimeType); + + content.DataUri = "data:text/plain;base64,VGhpcyBpcyBhIHRleHQgY29udGVudA=="; + Assert.True(content.CanRead); + Assert.Equal("text/plain", content.MimeType); + Assert.Equal("http://localhost/", content.Uri!.ToString()); + Assert.Equal(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA=="), content.Data!.Value.ToArray()); + } + + [Fact] + public void ItCanBecomeReadableAfterProvidingData() + { + var content = new BinaryContent(new Uri("http://localhost/")); + Assert.False(content.CanRead); + Assert.Equal("http://localhost/", content.Uri?.ToString()); + Assert.Null(content.MimeType); + + content.Data = new ReadOnlyMemory(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA==")); + Assert.True(content.CanRead); + Assert.Null(content.MimeType); + Assert.Equal("http://localhost/", content.Uri!.ToString()); + Assert.Equal(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA=="), content.Data!.Value.ToArray()); + } + + [Fact] + public void ItBecomesUnreadableAfterRemovingData() + { + var content = new BinaryContent(data: new ReadOnlyMemory(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA==")), mimeType: "text/plain"); + Assert.True(content.CanRead); + + content.Data = null; + Assert.False(content.CanRead); + Assert.Null(content.DataUri); + } + + [Fact] + public void ItBecomesUnreadableAfterRemovingDataUri() + { + var content = new BinaryContent(data: new ReadOnlyMemory(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA==")), mimeType: "text/plain"); + Assert.True(content.CanRead); + + content.DataUri = null; + Assert.False(content.CanRead); + Assert.Null(content.DataUri); + } + + [Fact] + public void GetDataUriWithoutMimeTypeShouldThrow() + { + // Arrange + var content = new BinaryContent + { + Data = new ReadOnlyMemory(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA==")) + }; + + // Act & Assert + Assert.Throws(() => content.DataUri); + } + + [Fact] + public void WhenProvidingDataUriToAnAlreadyExistingDataItOverridesAsExpected() + { + // Arrange + var content = new BinaryContent( + data: new ReadOnlyMemory(Convert.FromBase64String("VGhpcyBpcyBhIHRleHQgY29udGVudA==")), + mimeType: "text/plain") + { Uri = new Uri("http://localhost/") }; + + // Act + content.DataUri = "data:image/jpeg;base64,AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="; + + // Assert + Assert.Equal("image/jpeg", content.MimeType); + Assert.Equal("data:image/jpeg;base64,AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", content.DataUri); + Assert.NotNull(content.Data); + Assert.Equal(Convert.FromBase64String("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="), content.Data!.Value.ToArray()); + + // Don't change the referred Uri + Assert.Equal("http://localhost/", content.Uri?.ToString()); + } + + [Theory] + [InlineData( // Data always comes last in serialization + """{"Data": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "MimeType": "text/plain" }""", + """{"MimeType":"text/plain","Data":"AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8="}""")] + [InlineData( // Does not support non-readable content + """{"DataUri": "data:text/plain;base64,AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "unexpected": true }""", + "{}")] + [InlineData( // Only serializes the read/writeable properties + """{"DataUri": "data:text/plain;base64,AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "MimeType": "text/plain" }""", + """{"MimeType":"text/plain"}""")] + [InlineData( // Uri comes before mimetype + """{"MimeType": "text/plain", "Uri": "http://localhost/" }""", + """{"Uri":"http://localhost/","MimeType":"text/plain"}""")] + public void DeserializationAndSerializationBehaveAsExpected(string serialized, string expectedToString) + { + // Arrange + var content = JsonSerializer.Deserialize(serialized)!; + + // Act & Assert + var reSerialization = JsonSerializer.Serialize(content); + + Assert.Equal(expectedToString, reSerialization); + } + + [Theory] + // Other formats + [InlineData("http://localhost/", typeof(UriFormatException))] + [InlineData("about:blank", typeof(UriFormatException))] + [InlineData("file://c:\\temp", typeof(UriFormatException))] + [InlineData("invalid", typeof(UriFormatException))] + + // Data format validation errors + [InlineData("", typeof(UriFormatException))] // Empty data uri + [InlineData("data", typeof(UriFormatException))] // data missing colon + [InlineData("data:", typeof(UriFormatException))] // data missing comma + [InlineData("data:something,", typeof(UriFormatException))] // mime type without subtype + [InlineData("data:something;else,data", typeof(UriFormatException))] // mime type without subtype + [InlineData("data:type/subtype;parameterwithoutvalue;else,", typeof(UriFormatException))] // parameter without value + [InlineData("data:type/subtype;;parameter=value;else,", typeof(UriFormatException))] // parameter without value + [InlineData("data:type/subtype;parameter=va=lue;else,", typeof(UriFormatException))] // parameter with multiple = + [InlineData("data:type/subtype;=value;else,", typeof(UriFormatException))] // empty parameter name + + // Base64 Validation Errors + [InlineData("data:text;base64,something!", typeof(UriFormatException))] // Invalid base64 due to invalid character '!' + [InlineData("data:text/plain;base64,U29tZQ==\t", typeof(UriFormatException))] // Invalid base64 due to tab character + [InlineData("data:text/plain;base64,U29tZQ==\r", typeof(UriFormatException))] // Invalid base64 due to carriage return character + [InlineData("data:text/plain;base64,U29tZQ==\n", typeof(UriFormatException))] // Invalid base64 due to line feed character + [InlineData("data:text/plain;base64,U29t\r\nZQ==", typeof(UriFormatException))] // Invalid base64 due to carriage return and line feed characters + [InlineData("data:text/plain;base64,U29", typeof(UriFormatException))] // Invalid base64 due to missing padding + [InlineData("data:text/plain;base64,U29tZQ", typeof(UriFormatException))] // Invalid base64 due to missing padding + [InlineData("data:text/plain;base64,U29tZQ=", typeof(UriFormatException))] // Invalid base64 due to missing padding + public void ItThrowsOnInvalidDataUri(string path, Type exception) + { + var thrownException = Assert.Throws(exception, () => new BinaryContent(path)); + output.WriteLine(thrownException.Message); + } + + [Theory] + [InlineData("data:;parameter1=value1,", "{}", """{"data-uri-parameter1":"value1"}""")] // Should create extra data + [InlineData("data:;parameter1=value1,", """{"Metadata":{"data-uri-parameter1":"should override me"}}""", """{"data-uri-parameter1":"value1"}""")] // Should override existing data + [InlineData("data:;parameter1=value1,", """{"Metadata":{"data-uri-parameter2":"value2"}}""", """{"data-uri-parameter1":"value1","data-uri-parameter2":"value2"}""")] // Should merge existing data with new data + [InlineData("data:;parameter1=value1;parameter2=value2,data", """{"Metadata":{"data-uri-parameter2":"should override me"}}""", """{"data-uri-parameter1":"value1","data-uri-parameter2":"value2"}""")] // Should merge existing data with new data + [InlineData("data:image/jpeg;parameter1=value1;parameter2=value2;base64,data", """{"Metadata":{"data-uri-parameter2":"should override me"}}""", """{"data-uri-parameter1":"value1","data-uri-parameter2":"value2"}""")] // Should merge existing data with new data + [InlineData("data:image/jpeg;parameter1=value1;parameter2=value2;base64,data", """{"Metadata":{"data-uri-parameter3":"existing data", "data-uri-parameter2":"should override me"}}""", """{"data-uri-parameter1":"value1","data-uri-parameter2":"value2","data-uri-parameter3":"existing data"}""")] // Should keep previous metadata + public void DataUriConstructorWhenProvidingParametersUpdatesMetadataAsExpected(string path, string startingSerializedBinaryContent, string expectedSerializedMetadata) + { + // Arrange + var content = JsonSerializer.Deserialize(startingSerializedBinaryContent)!; + content.DataUri = path; + + var expectedMetadata = JsonSerializer.Deserialize>(expectedSerializedMetadata)!; + + // Act & Assert + Assert.Equal(expectedMetadata.Count, content.Metadata!.Count); + foreach (var kvp in expectedMetadata) + { + Assert.True(content.Metadata.ContainsKey(kvp.Key)); + Assert.Equal(kvp.Value?.ToString(), content.Metadata[kvp.Key]?.ToString()); + } + } + + [Fact] + public void ItPreservePreviousMetadataForParameterizedDataUri() + { + // Arrange + var content = new BinaryContent + { + Metadata = new Dictionary + { + { "key1", "value1" }, + { "key2", "value2" } + } + }; + + // Act + content.DataUri = "data:;parameter1=parametervalue1;parameter2=parametervalue2;base64,data"; + + // Assert + Assert.Equal(4, content.Metadata!.Count); + Assert.Equal("value1", content.Metadata["key1"]?.ToString()); + Assert.Equal("value2", content.Metadata["key2"]?.ToString()); + Assert.Equal("parametervalue1", content.Metadata["data-uri-parameter1"]?.ToString()); + Assert.Equal("parametervalue2", content.Metadata["data-uri-parameter2"]?.ToString()); + } + + [Fact] + public void DeserializesParameterizedDataUriAsExpected() + { + // Arrange + var content = new BinaryContent + { + Data = new ReadOnlyMemory(Convert.FromBase64String("U29tZSBkYXRh")), + MimeType = "application/json", + Metadata = new Dictionary + { + { "data-uri-parameter1", "value1" }, + { "data-uri-parameter2", "value2" } + } + }; + + var expectedDataUri = "data:application/json;parameter1=value1;parameter2=value2;base64,U29tZSBkYXRh"; + + // Act & Assert + Assert.Equal(expectedDataUri, content.DataUri); + } + + [Theory] + [InlineData("data:application/octet-stream;utf8,01-02-03-04")] + [InlineData("data:application/json,01-02-03-04")] + [InlineData("data:,01-02-03-04")] + public void ReturnUTF8EncodedWhenDataIsNotBase64(string path) + { + // Arrange + var content = new BinaryContent(path); + + // Act & Assert + Assert.Equal("01-02-03-04", Encoding.UTF8.GetString(content.Data!.Value.ToArray())); + } +} diff --git a/dotnet/src/SemanticKernel.UnitTests/Contents/ChatMessageContentTests.cs b/dotnet/src/SemanticKernel.UnitTests/Contents/ChatMessageContentTests.cs index f2034a896407..a25376128f2d 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Contents/ChatMessageContentTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Contents/ChatMessageContentTests.cs @@ -158,54 +158,18 @@ public void EncodingPropertyGetterShouldReturnEncodingOfTextContentItem() public void ItCanBeSerializeAndDeserialized() { // Arrange - var items = new ChatMessageContentItemCollection - { - new TextContent("content-1", "model-1", metadata: new Dictionary() - { - ["metadata-key-1"] = "metadata-value-1" - }) { MimeType = "mime-type-1" }, - new ImageContent(new Uri("https://fake-random-test-host:123"), "model-2", metadata: new Dictionary() - { - ["metadata-key-2"] = "metadata-value-2" - }) { MimeType = "mime-type-2" }, - new BinaryContent(new BinaryData(new[] { 1, 2, 3 }), "model-3", metadata: new Dictionary() - { - ["metadata-key-3"] = "metadata-value-3" - }) { MimeType = "mime-type-3" }, - new AudioContent(new BinaryData(new[] { 3, 2, 1 }), "model-4", metadata: new Dictionary() - { - ["metadata-key-4"] = "metadata-value-4" - }) { MimeType = "mime-type-4" }, - new ImageContent(new BinaryData(new[] { 2, 1, 3 }), "model-5", metadata: new Dictionary() - { - ["metadata-key-5"] = "metadata-value-5" - }) { MimeType = "mime-type-5" }, - new TextContent("content-6", "model-6", metadata: new Dictionary() - { - ["metadata-key-6"] = "metadata-value-6" - }) { MimeType = "mime-type-6" }, + ChatMessageContentItemCollection items = [ + new TextContent("content-1", "model-1", metadata: new Dictionary() { ["metadata-key-1"] = "metadata-value-1" }) { MimeType = "mime-type-1" }, + new ImageContent(new Uri("https://fake-random-test-host:123")) { ModelId = "model-2", MimeType = "mime-type-2", Metadata = new Dictionary() { ["metadata-key-2"] = "metadata-value-2" } }, + new BinaryContent(new BinaryData(new[] { 1, 2, 3 }), mimeType: "mime-type-3") { ModelId = "model-3", Metadata = new Dictionary() { ["metadata-key-3"] = "metadata-value-3" } }, + new AudioContent(new BinaryData(new[] { 3, 2, 1 }), mimeType: "mime-type-4") { ModelId = "model-4", Metadata = new Dictionary() { ["metadata-key-4"] = "metadata-value-4" } }, + new ImageContent(new BinaryData(new[] { 2, 1, 3 }), mimeType: "mime-type-5") { ModelId = "model-5", Metadata = new Dictionary() { ["metadata-key-5"] = "metadata-value-5" } }, + new TextContent("content-6", "model-6", metadata: new Dictionary() { ["metadata-key-6"] = "metadata-value-6" }) { MimeType = "mime-type-6" }, new FunctionCallContent("function-name", "plugin-name", "function-id", new KernelArguments { ["parameter"] = "argument" }), new FunctionResultContent(new FunctionCallContent("function-name", "plugin-name", "function-id"), "function-result"), - new FileReferenceContent( - fileId: "file-id-1", - modelId: "model-7", - metadata: new Dictionary() - { - ["metadata-key-7"] = "metadata-value-7" - }), - new AnnotationContent( - modelId: "model-8", - metadata: new Dictionary() - { - ["metadata-key-8"] = "metadata-value-8" - }) - { - FileId = "file-id-2", - StartIndex = 2, - EndIndex = 24, - Quote = "quote-8" - }, - }; + new FileReferenceContent(fileId: "file-id-1") { ModelId = "model-7", Metadata = new Dictionary() { ["metadata-key-7"] = "metadata-value-7" } }, + new AnnotationContent() { ModelId = "model-8", FileId = "file-id-2", StartIndex = 2, EndIndex = 24, Quote = "quote-8", Metadata = new Dictionary() { ["metadata-key-8"] = "metadata-value-8" } } + ]; // Act var chatMessageJson = JsonSerializer.Serialize(new ChatMessageContent(AuthorRole.User, items: items, "message-model", metadata: new Dictionary() @@ -253,7 +217,7 @@ public void ItCanBeSerializeAndDeserialized() var binaryContent = deserializedMessage.Items[2] as BinaryContent; Assert.NotNull(binaryContent); - Assert.True(binaryContent.Content?.Span.SequenceEqual(new BinaryData(new[] { 1, 2, 3 }))); + Assert.True(binaryContent.Data!.Value.Span.SequenceEqual(new BinaryData(new[] { 1, 2, 3 }))); Assert.Equal("model-3", binaryContent.ModelId); Assert.Equal("mime-type-3", binaryContent.MimeType); Assert.NotNull(binaryContent.Metadata); @@ -300,7 +264,7 @@ public void ItCanBeSerializeAndDeserialized() Assert.NotNull(functionResultContent); Assert.Equal("function-result", functionResultContent.Result?.ToString()); Assert.Equal("function-name", functionResultContent.FunctionName); - Assert.Equal("function-id", functionResultContent.Id); + Assert.Equal("function-id", functionResultContent.CallId); Assert.Equal("plugin-name", functionResultContent.PluginName); var fileReferenceContent = deserializedMessage.Items[8] as FileReferenceContent; diff --git a/dotnet/src/SemanticKernel.UnitTests/Contents/FunctionResultContentTests.cs b/dotnet/src/SemanticKernel.UnitTests/Contents/FunctionResultContentTests.cs index fe10c4aca308..6229f98863fe 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Contents/FunctionResultContentTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Contents/FunctionResultContentTests.cs @@ -22,7 +22,7 @@ public void ItShouldHaveFunctionIdInitialized() var sut = new FunctionResultContent(this._callContent, "result"); // Assert - Assert.Equal("id", sut.Id); + Assert.Equal("id", sut.CallId); } [Fact] @@ -82,7 +82,7 @@ public void ItShouldBeSerializableAndDeserializable() // Assert Assert.NotNull(deserializedSut); - Assert.Equal(sut.Id, deserializedSut.Id); + Assert.Equal(sut.CallId, deserializedSut.CallId); Assert.Equal(sut.PluginName, deserializedSut.PluginName); Assert.Equal(sut.FunctionName, deserializedSut.FunctionName); Assert.Equal(sut.Result, deserializedSut.Result?.ToString()); diff --git a/dotnet/src/SemanticKernel.UnitTests/Contents/ImageContentTests.cs b/dotnet/src/SemanticKernel.UnitTests/Contents/ImageContentTests.cs index 03c5604e3637..14f86451cf71 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Contents/ImageContentTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Contents/ImageContentTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Text; +using System.Text.Json; using Microsoft.SemanticKernel; using Xunit; @@ -23,23 +25,23 @@ public void ToStringForUriReturnsString() var result2 = content2.ToString(); // Assert - Assert.Empty(result1); - Assert.Equal("https://endpoint/", result2); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", result1); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", result2); } [Fact] - public void ToStringForDataUriReturnsDataUriString() + public void ToStringForDataUriReturnsTypeString() { // Arrange var data = BinaryData.FromString("this is a test"); - var content1 = new ImageContent(data) { MimeType = "text/plain" }; + var content1 = new ImageContent(data, "text/plain"); // Act var result1 = content1.ToString(); var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; // Assert - Assert.Equal(dataUriToExpect, result1); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", result1); } [Fact] @@ -47,18 +49,14 @@ public void ToStringForUriAndDataUriReturnsDataUriString() { // Arrange var data = BinaryData.FromString("this is a test"); - var content1 = new ImageContent(data) - { - MimeType = "text/plain", - Uri = new Uri("https://endpoint/") - }; + var content1 = new ImageContent(data, "text/plain") { Uri = new Uri("https://endpoint/") }; // Act var result1 = content1.ToString(); var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; // Assert - Assert.Equal(dataUriToExpect, result1); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", result1); } [Fact] @@ -68,23 +66,24 @@ public void CreateForEmptyDataUriThrows() var data = BinaryData.Empty; // Assert - Assert.Throws(() => new ImageContent(data) { MimeType = "text/plain" }); + Assert.Throws(() + => new ImageContent(data, "text/plain")); } [Fact] - public void ToStringForDataUriFromBytesReturnsDataUriString() + public void ToStringForDataUriFromBytesReturnsType() { // Arrange var bytes = System.Text.Encoding.UTF8.GetBytes("this is a test"); var data = BinaryData.FromBytes(bytes); - var content1 = new ImageContent(data) { MimeType = "text/plain" }; + var content1 = new ImageContent(data, "text/plain"); // Act var result1 = content1.ToString(); var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; // Assert - Assert.Equal(dataUriToExpect, result1); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", result1); } [Fact] @@ -93,29 +92,185 @@ public void ToStringForDataUriFromStreamReturnsDataUriString() // Arrange using var ms = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes("this is a test")); var data = BinaryData.FromStream(ms); - var content1 = new ImageContent(data) { MimeType = "text/plain" }; + var content1 = new ImageContent(data, "text/plain"); // Act var result1 = content1.ToString(); var dataUriToExpect = $"data:text/plain;base64,{Convert.ToBase64String(data.ToArray())}"; // Assert - Assert.Equal(dataUriToExpect, result1); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", result1); + } + + [Fact] + public void DataConstructorWhenDataIsEmptyShouldThrow() + { + // Arrange + using var ms = new System.IO.MemoryStream(System.Text.Encoding.UTF8.GetBytes("this is a test")); + + var data = BinaryData.FromStream(ms); // Assert throws if mediatype is null - Assert.Throws(() => new ImageContent(BinaryData.FromStream(ms)) { MimeType = null! }); + Assert.Throws(() => new ImageContent(BinaryData.FromStream(ms), mimeType: null)); } [Fact] - public void InMemoryImageWithoutMediaTypeReturnsEmptyString() + public void ToStringInMemoryImageWithoutMediaTypeReturnsType() { // Arrange - var sut = new ImageContent(new byte[] { 1, 2, 3 }) { MimeType = null }; + var sut = new ImageContent(new byte[] { 1, 2, 3 }, mimeType: null); // Act var dataUrl = sut.ToString(); // Assert - Assert.Empty(dataUrl); + Assert.Equal($"Microsoft.SemanticKernel.{nameof(ImageContent)}", dataUrl?.ToString()); + } + + // Ensure retrocompatibility with ImageContent Pre-BinaryContent Version + + [Theory] + [InlineData("", null, $"Microsoft.SemanticKernel.{nameof(ImageContent)}")] + [InlineData(null, null, $"Microsoft.SemanticKernel.{nameof(ImageContent)}")] + [InlineData("", "http://localhost:9090/", $"Microsoft.SemanticKernel.{nameof(ImageContent)}")] + [InlineData(null, "http://localhost:9090/", $"Microsoft.SemanticKernel.{nameof(ImageContent)}")] + [InlineData("image/png", null, $"Microsoft.SemanticKernel.{nameof(ImageContent)}")] + [InlineData("image/png", "http://localhost:9090", $"Microsoft.SemanticKernel.{nameof(ImageContent)}")] + public void ToStringShouldReturn(string? mimeType, string? path, string expectedToString) + { + // Arrange + var bytes = Encoding.UTF8.GetBytes("this is a test"); + var data = BinaryData.FromBytes(bytes); + var content1 = new ImageContent(data, mimeType); + if (path is not null) + { + content1.Uri = new Uri(path); + } + + // Act + var result1 = content1.ToString(); + + // Assert + Assert.Equal(expectedToString, result1); + } + + [Fact] + public void UpdatingUriPropertyShouldReturnAsExpected() + { + // Arrange + var data = BinaryData.FromString("this is a test"); + var content = new ImageContent(data, "text/plain"); + + // Act + var serializeBefore = JsonSerializer.Serialize(content); + + // Changing the Uri to a absolute file /foo.txt path + content.Uri = new Uri("file:///foo.txt"); + content.MimeType = "image/jpeg"; + + var serializeAfter = JsonSerializer.Serialize(content); + + // Assert + Assert.Equal("""{"MimeType":"text/plain","Data":"dGhpcyBpcyBhIHRlc3Q="}""", serializeBefore); + Assert.Equal("""{"Uri":"file:///foo.txt","MimeType":"image/jpeg","Data":"dGhpcyBpcyBhIHRlc3Q="}""", serializeAfter); + + // Uri behaves independently of other properties + Assert.Equal("file:///foo.txt", content.Uri?.ToString()); + + // Data and MimeType remain the same + Assert.Equal(Convert.FromBase64String("dGhpcyBpcyBhIHRlc3Q="), content.Data!.Value.ToArray()); + Assert.Equal(data.ToArray(), content.Data!.Value.ToArray()); + } + + [Fact] + public void UpdatingMimeTypePropertyShouldReturnAsExpected() + { + // Arrange + var data = BinaryData.FromString("this is a test"); + var content = new ImageContent(data, "text/plain"); + + // Act + var toStringBefore = content.ToString(); + + // Changing the mimetype to image/jpeg in the DataUri + Assert.Equal("data:text/plain;base64,dGhpcyBpcyBhIHRlc3Q=", content.DataUri); + + content.MimeType = "application/json"; + Assert.Equal("data:application/json;base64,dGhpcyBpcyBhIHRlc3Q=", content.DataUri); + Assert.Null(content.Uri); // Uri behaves independently of other properties, was not set, keeps null. + Assert.Equal(Convert.FromBase64String("dGhpcyBpcyBhIHRlc3Q="), content.Data!.Value.ToArray()); + Assert.Equal(data.ToArray(), content.Data!.Value.ToArray()); + Assert.Equal("application/json", content.MimeType); + } + + [Fact] + public void UpdateDataPropertyShouldReturnAsExpected() + { + // Arrange + var dataUriBefore = "data:text/plain;base64,dGhpcyBpcyBhIHRlc3Q="; + var content = new ImageContent(dataUriBefore); + + // Act + var newData = BinaryData.FromString("this is a new test"); + dataUriBefore = content.DataUri!; + content.Data = newData; + + // Assert + Assert.Equal("data:text/plain;base64,dGhpcyBpcyBhIHRlc3Q=", dataUriBefore); + Assert.Equal("data:text/plain;base64,dGhpcyBpcyBhIG5ldyB0ZXN0", content.DataUri); + Assert.Null(content.Uri); // Uri behaves independently of other properties, was not set, keeps null. + Assert.Equal("text/plain", content.MimeType); // MimeType remain the same as it was not set + Assert.Equal(Convert.FromBase64String("dGhpcyBpcyBhIG5ldyB0ZXN0"), content.Data!.Value.ToArray()); // Data is updated + } + + [Fact] + public void EmptyConstructorSerializationAndDeserializationAsExpected() + { + var content = new ImageContent(); + var serialized = JsonSerializer.Serialize(content); + var deserialized = JsonSerializer.Deserialize(serialized); + + Assert.Equal("{}", serialized); + + Assert.NotNull(deserialized); + Assert.Null(deserialized.Uri); + Assert.Null(deserialized.Data); + Assert.Null(deserialized.MimeType); + Assert.Null(deserialized.InnerContent); + Assert.Null(deserialized.ModelId); + Assert.Null(deserialized.Metadata); + } + + [Theory] + [InlineData("http://localhost:9090/")] + [InlineData(null)] + public void UriConstructorSerializationAndDeserializationAsExpected(string? path) + { +#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type. +#pragma warning disable CS8604 // Possible null reference argument. + Uri uri = path is not null ? new Uri(path) : null; + + var content = new ImageContent(uri); + var serialized = JsonSerializer.Serialize(content); + var deserialized = JsonSerializer.Deserialize(serialized); + + if (uri is null) + { + Assert.Equal("{}", serialized); + } + else + { + Assert.Equal($"{{\"Uri\":\"{uri}\"}}", serialized); + } + + Assert.NotNull(deserialized); + Assert.Equal(uri, deserialized.Uri); + Assert.Null(deserialized.Data); + Assert.Null(deserialized.MimeType); + Assert.Null(deserialized.InnerContent); + Assert.Null(deserialized.ModelId); + Assert.Null(deserialized.Metadata); +#pragma warning restore CS8604 // Possible null reference argument. +#pragma warning restore CS8600 // Converting null literal or possible null value to non-nullable type. } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Utilities/DataUriParserTests.cs b/dotnet/src/SemanticKernel.UnitTests/Utilities/DataUriParserTests.cs new file mode 100644 index 000000000000..2251450cfc05 --- /dev/null +++ b/dotnet/src/SemanticKernel.UnitTests/Utilities/DataUriParserTests.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.SemanticKernel.Text; +using Xunit; +using Xunit.Abstractions; + +#pragma warning disable CA1054 // URI-like parameters should not be strings +#pragma warning disable CA1055 // URI-like parameters should not be strings +#pragma warning disable CA1056 // URI-like parameters should not be strings + +namespace SemanticKernel.UnitTests.Utilities; + +public class DataUriParserTests(ITestOutputHelper output) +{ + [Theory] + [InlineData("data:,", "text/plain", "", null, "{}")] // Minimum valid data URI + [InlineData("data:,A%20brief%20note", "text/plain", "A%20brief%20note", null, "{}")] // Example in RFC 2397 Doc + [InlineData("data:text/plain;charset=iso-8859-7,%be%fg%be", "text/plain", "%be%fg%be", null, """{"charset":"iso-8859-7"}""")] // Example in RFC 2397 Doc + [InlineData("""data:image/gif;base64,R0lGODdhMAAwAPAAAAAAAP///ywAAAAAMAAwAAAC8IyPqcvt""", "image/gif", "R0lGODdhMAAwAPAAAAAAAP///ywAAAAAMAAwAAAC8IyPqcvt", "base64", "{}")] // Example in RFC 2397 Doc + [InlineData("data:text/vnd-example+xyz;foo=bar;base64,R0lGODdh", "text/vnd-example+xyz", "R0lGODdh", "base64", """{"foo":"bar"}""")] + [InlineData("data:application/octet-stream;base64,AQIDBA==", "application/octet-stream", "AQIDBA==", "base64", "{}")] + [InlineData("data:text/plain;charset=UTF-8;page=21,the%20data:1234,5678", "text/plain", "the%20data:1234,5678", null, """{"charset":"UTF-8","page":"21"}""")] + [InlineData("data:image/svg+xml;utf8,", "image/svg+xml", "", "utf8", "{}")] + [InlineData("data:;charset=UTF-8,the%20data", "text/plain", "the%20data", null, """{"charset":"UTF-8"}""")] + [InlineData("data:text/vnd-example+xyz;foo=;base64,R0lGODdh", "text/vnd-example+xyz", "R0lGODdh", "base64", """{"foo":""}""")] + public void ItCanParseDataUri(string dataUri, string? expectedMimeType, string expectedData, string? expectedDataFormat, string serializedExpectedParameters) + { + var parsed = DataUriParser.Parse(dataUri); + var expectedParameters = JsonSerializer.Deserialize>(serializedExpectedParameters); + + Assert.Equal(expectedMimeType, parsed.MimeType); + Assert.Equal(expectedData, parsed.Data); + Assert.Equal(expectedDataFormat, parsed.DataFormat); + Assert.Equal(expectedParameters!.Count, parsed.Parameters.Count); + if (expectedParameters.Count > 0) + { + foreach (var kvp in expectedParameters) + { + Assert.True(parsed.Parameters.ContainsKey(kvp.Key)); + Assert.Equal(kvp.Value, parsed.Parameters[kvp.Key]); + } + } + } + + [Theory] + // Data format validation errors + [InlineData("", typeof(ArgumentException))] + [InlineData(null, typeof(ArgumentNullException))] + [InlineData("data", typeof(UriFormatException))] // data missing colon + [InlineData("data:", typeof(UriFormatException))] // data missing comma + [InlineData("data:something,", typeof(UriFormatException))] // mime type without subtype + [InlineData("data:something;else,data", typeof(UriFormatException))] // mime type without subtype + [InlineData("data:type/subtype;parameterwithoutvalue;else,", typeof(UriFormatException))] // parameter without value + [InlineData("data:type/subtype;;parameter=value;else,", typeof(UriFormatException))] // parameter without value + [InlineData("data:type/subtype;parameter=va=lue;else,", typeof(UriFormatException))] // parameter with multiple = + [InlineData("data:type/subtype;=value;else,", typeof(UriFormatException))] // empty parameter name + // Base64 Validation Errors + [InlineData("data:text;base64,something!", typeof(UriFormatException))] // Invalid base64 due to invalid character '!' + [InlineData("data:text/plain;base64,U29tZQ==\t", typeof(UriFormatException))] // Invalid base64 due to tab character + [InlineData("data:text/plain;base64,U29tZQ==\r", typeof(UriFormatException))] // Invalid base64 due to carriage return character + [InlineData("data:text/plain;base64,U29tZQ==\n", typeof(UriFormatException))] // Invalid base64 due to line feed character + [InlineData("data:text/plain;base64,U29t\r\nZQ==", typeof(UriFormatException))] // Invalid base64 due to carriage return and line feed characters + [InlineData("data:text/plain;base64,U29", typeof(UriFormatException))] // Invalid base64 due to missing padding + [InlineData("data:text/plain;base64,U29tZQ", typeof(UriFormatException))] // Invalid base64 due to missing padding + [InlineData("data:text/plain;base64,U29tZQ=", typeof(UriFormatException))] // Invalid base64 due to missing padding + public void ItThrowsOnInvalidDataUri(string? dataUri, Type exception) + { + var thrownException = Assert.Throws(exception, () => DataUriParser.Parse(dataUri)); + output.WriteLine(thrownException.Message); + } +}