From 8fe72eda557c6512299b3566d376220d204c3e59 Mon Sep 17 00:00:00 2001 From: marken Date: Wed, 2 Oct 2024 06:50:40 +0800 Subject: [PATCH 1/4] chore: Support file_search tool call in StepDetails of RunStep --- .../run/RunStepDetails.kt | 80 ++++++++++++++++++- 1 file changed, 79 insertions(+), 1 deletion(-) diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/run/RunStepDetails.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/run/RunStepDetails.kt index 2c7220a7..20f4902b 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/run/RunStepDetails.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/run/RunStepDetails.kt @@ -48,7 +48,7 @@ public data class ToolCallStepDetails( /** * An array of tool calls the run step was involved in. * These can be associated with one of three types of tools: - * [ToolCallStep.CodeInterpreter], [ToolCallStep.RetrievalTool], or [ToolCallStep.FunctionTool]. + * [ToolCallStep.CodeInterpreter], [ToolCallStep.RetrievalTool], [ToolCallStep.FunctionTool] or [ToolCallStep.FileSearch]. */ @SerialName("tool_calls") public val toolCalls: List? = null, ) : RunStepDetails @@ -98,6 +98,84 @@ public sealed interface ToolCallStep { */ @SerialName("function") public val function: FunctionToolCallStep, ) : ToolCallStep + + @BetaOpenAI + @Serializable + @SerialName("file_search") + public data class FileSearch( + /** + * The ID of the tool call. + */ + @SerialName("id") public val id: ToolCallStepId, + /** + * The File Search tool call definition. + */ + @SerialName("file_search") public val fileSearch: FileSearchToolCallStep, + ) : ToolCallStep +} + +@BetaOpenAI +@Serializable +public data class FileSearchToolCallStep( + /** + * The ranking options for the file search. + */ + @SerialName("ranking_options") val rankingOptions: RankingOptions, + /** + * The results of the file search. + */ + @SerialName("results") val results: List, +) + +/** + * RankingOption for FileSearch Call Step + */ +@BetaOpenAI +@Serializable +public data class RankingOptions( + /** + * The ranker used for the file search. + */ + @SerialName("ranker") val ranker: String, + /** + * The score threshold for the file search. + * All values must be a floating point number between 0 and 1. + */ + @SerialName("score_threshold") val scoreThreshold: Double +) + +/** + * The results of the file search. + */ +@BetaOpenAI +@Serializable +public data class FileSearchResult( + /** + * The ID of the file that result was found in. + */ + @SerialName("file_id") val fileID: String, + /** + * The name of the file that result was found in. + */ + @SerialName("file_name") val fileName: String, + /** + * The score of the result. + * All values must be a floating point number between 0 and 1. + */ + @SerialName("score") val score: Double, + /** + * The content of the result that was found. + * The content is only included if requested via the include query parameter. + */ + @SerialName("content") val content: List? = null, +) { + + @BetaOpenAI + @Serializable + @SerialName("text") + public data class TextContent( + @SerialName("text") public val text: String + ) } @BetaOpenAI From 0990ab7cebb62873de35cf62a07c67d8800ed1d4 Mon Sep 17 00:00:00 2001 From: marken Date: Wed, 2 Oct 2024 07:06:49 +0800 Subject: [PATCH 2/4] chore: Support ImageUrl for MessageContent --- .../message/ImageFile.kt | 23 ++++++++++++- .../message/MessageContent.kt | 32 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/ImageFile.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/ImageFile.kt index 8e4204c6..2dfee04d 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/ImageFile.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/ImageFile.kt @@ -12,5 +12,26 @@ public data class ImageFile( /** * The File ID of the image in the message content. */ - @SerialName("file_id") val fileId: FileId + @SerialName("file_id") val fileId: FileId, + /** + * Optional Defaults to auto + * Specifies the detail level of the image if specified by the user. low uses fewer tokens, you can opt in to high resolution using high. + */ + @SerialName("detail") val detail: String? = null, ) + +/** + * Image content part data. + */ +@Serializable +public data class ImageURL( + /** + * Either a URL of the image or the base64 encoded image data. + */ + @SerialName("url") val url: String, + + /** + * Specifies the detail level of the image. + */ + @SerialName("detail") val detail: String? = null, +) \ No newline at end of file diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt index b7dd8bed..f57e7328 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt @@ -31,15 +31,41 @@ public sealed interface MessageContent { @BetaOpenAI @Serializable @SerialName("image_file") - public data class Image( + public data class ImageFile( /** * The File ID of the image in the message content. */ - @SerialName("image_file") val imageFile: ImageFile + @SerialName("image_file") val imageFile: com.aallam.openai.api.message.ImageFile + ) : MessageContent + + /** + * References an image File in the content of a message. + */ + @BetaOpenAI + @Serializable + @SerialName("image_url") + public data class ImageURL( + /** + * The Image Url of the image in the message content. + */ + @SerialName("image_url") val imageUrl: com.aallam.openai.api.message.ImageURL ) : MessageContent -} + /** + * The refusal content generated by the assistant. + */ + @BetaOpenAI + @Serializable + @SerialName("refusal") + public data class Refusal( + /** + * The data that makes up the refusal. + */ + @SerialName("refusal") val refusal: String + ) : MessageContent + +} /** * The text content of the message value and annotations. */ From 0b6c5bba1f5afc4bb241cfabc4da0b82b1fc06d1 Mon Sep 17 00:00:00 2001 From: marken Date: Wed, 2 Oct 2024 07:08:10 +0800 Subject: [PATCH 3/4] chore: Support the array content for MessageRequest --- .../message/MessageRequest.kt | 69 ++++++++-- .../message/MessageRequestContent.kt | 124 ++++++++++++++++++ .../MessageRequestContentSerializer.kt | 28 ++++ 3 files changed, 213 insertions(+), 8 deletions(-) create mode 100644 openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt create mode 100644 openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/internal/MessageRequestContentSerializer.kt diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt index 4cfec89f..75b1da6b 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt @@ -2,6 +2,8 @@ package com.aallam.openai.api.message import com.aallam.openai.api.BetaOpenAI import com.aallam.openai.api.core.Role +import com.aallam.openai.api.message.MessageRequestContent.ListContent +import com.aallam.openai.api.message.MessageRequestContent.TextContent import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @@ -19,7 +21,7 @@ public class MessageRequest( /** * The content of the message. */ - @SerialName("content") public val content: String, + @SerialName("content") public val messageContent: MessageRequestContent, /** * A list of files attached to the message. @@ -32,7 +34,37 @@ public class MessageRequest( * Keys can be a maximum of 64 characters long, and values can be a maximum of 512 characters long. */ @SerialName("metadata") public val metadata: Map? = null, -) +) { + public constructor( + role: Role, + content: String, + attachments: List? = null, + metadata: Map? = null, + ) : this( + role = role, + messageContent = TextContent(content), + attachments = attachments, + metadata = metadata + ) + + public constructor( + role: Role, + content: List, + attachments: List? = null, + metadata: Map? = null, + ) : this( + role = role, + messageContent = ListContent(content), + attachments = attachments, + metadata = metadata + ) + + public val content: String + get() = when (messageContent) { + is TextContent -> messageContent.content + else -> error("Content is not text") + } +} /** * A message request builder. @@ -68,10 +100,31 @@ public class MessageRequestBuilder { */ public var metadata: Map? = null - public fun build(): MessageRequest = MessageRequest( - role = requireNotNull(role) { "role is required" }, - content = requireNotNull(content) { "content is required" }, - attachments = attachments, - metadata = metadata - ) + /** + * The contents of the assistant request message. + */ + internal val parts = mutableListOf() + + /** + * The contents of the message. + */ + public fun content(block: MessageRequestPartBuilder.() -> Unit) { + this.parts += MessageRequestPartBuilder().apply(block).build() + } + + /** + * Create [MessageRequest] instance. + */ + public fun build(): MessageRequest { + require(!(content != null && parts.isNotEmpty())) { "Cannot set both content string and content parts" } + val messageContent = content?.let { MessageRequestContent.TextContent(it) } + ?: ListContent(parts) + return MessageRequest( + role = requireNotNull(role) { "role is required " }, + messageContent = messageContent, + attachments = attachments, + metadata = metadata + ) + } } + diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt new file mode 100644 index 00000000..2f8c5a7d --- /dev/null +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt @@ -0,0 +1,124 @@ +package com.aallam.openai.api.message + +import com.aallam.openai.api.BetaOpenAI +import com.aallam.openai.api.OpenAIDsl +import com.aallam.openai.api.file.FileId +import com.aallam.openai.api.message.MessageRequestPart.ImageFilePart +import com.aallam.openai.api.message.MessageRequestPart.ImageUrlPart +import com.aallam.openai.api.message.MessageRequestPart.TextPart +import com.aallam.openai.api.message.internal.MessageRequestContentSerializer +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable +import kotlin.jvm.JvmInline + +/** + * The contents of the assistant request message. + */ +@BetaOpenAI +@Serializable(with = MessageRequestContentSerializer::class) +public sealed interface MessageRequestContent { + /** + * The assistant message content as text. + */ + @BetaOpenAI + @JvmInline + @Serializable + public value class TextContent(public val content: String) : MessageRequestContent + + /** + * The assistant message content as a list of content parts. + */ + @BetaOpenAI + @JvmInline + @Serializable + public value class ListContent(public val content: List) : MessageRequestContent +} + + +@BetaOpenAI +@Serializable +public sealed interface MessageRequestPart { + /** + * The content of the message as text. + */ + @BetaOpenAI + @Serializable + @SerialName("text") + public data class TextPart( + /** + * The text content of the message value and annotations. + */ + @SerialName("text") val text: String + ) : MessageRequestPart + + /** + * References an image File in the content of a message. + */ + @BetaOpenAI + @Serializable + @SerialName("image_file") + public data class ImageFilePart( + /** + * The Image file object of the image in the message content. + */ + @SerialName("image_file") val imageFile: ImageFile + ) : MessageRequestPart + + /** + * References an image url in the content of a message. + */ + @BetaOpenAI + @Serializable + @SerialName("image_url") + public data class ImageUrlPart( + /** + * The Image url object of the image in the message content. + */ + @SerialName("image_url") val imageUrl: ImageURL + ) : MessageRequestPart + +} + +@OptIn(BetaOpenAI::class) +@OpenAIDsl +public class MessageRequestPartBuilder { + + private val parts = mutableListOf() + + /** + * Text content part. + * + * @param text the text content. + */ + public fun text(text: String) { + this.parts += TextPart(text) + } + + /** + * Image file content part. + * + * @param id the image file. + * @param detail the image detail. + */ + public fun imageFile(id: FileId, detail: String? = null) { + this.parts += ImageFilePart(ImageFile(id, detail)) + } + + /** + * Image url content part. + * + * @param url the image url. + * @param detail the image detail. + */ + public fun imageUrl(url: String, detail: String? = null) { + this.parts += ImageUrlPart(ImageURL(url, detail)) + } + + + /** + * Create a list of [MessageRequestPart]s. + */ + public fun build(): List { + return parts + } +} \ No newline at end of file diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/internal/MessageRequestContentSerializer.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/internal/MessageRequestContentSerializer.kt new file mode 100644 index 00000000..74db0871 --- /dev/null +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/internal/MessageRequestContentSerializer.kt @@ -0,0 +1,28 @@ +package com.aallam.openai.api.message.internal + + +import com.aallam.openai.api.BetaOpenAI +import com.aallam.openai.api.message.MessageRequestContent +import com.aallam.openai.api.message.MessageRequestContent.ListContent +import com.aallam.openai.api.message.MessageRequestContent.TextContent +import kotlinx.serialization.DeserializationStrategy +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.JsonContentPolymorphicSerializer +import kotlinx.serialization.json.JsonElement +import kotlinx.serialization.json.JsonPrimitive + +/** + * Serializer for [MessageRequestContent]. + */ +@OptIn(BetaOpenAI::class) +internal class MessageRequestContentSerializer : + JsonContentPolymorphicSerializer(MessageRequestContent::class) { + override fun selectDeserializer(element: JsonElement): DeserializationStrategy { + return when (element) { + is JsonPrimitive -> TextContent.serializer() + is JsonArray -> ListContent.serializer() + else -> throw SerializationException("Unsupported JSON element: $element") + } + } +} From 1073f7df6073253e6ebfe9c421cba9171c106bc6 Mon Sep 17 00:00:00 2001 From: marken Date: Fri, 28 Feb 2025 16:18:16 +0800 Subject: [PATCH 4/4] chore: update to 4.0.1 --- .../com.aallam.openai.api/message/MessageContent.kt | 3 ++- .../com.aallam.openai.api/message/MessageRequest.kt | 8 +++----- .../message/MessageRequestContent.kt | 4 +--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt index f57e7328..6e6fecfd 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt @@ -31,7 +31,7 @@ public sealed interface MessageContent { @BetaOpenAI @Serializable @SerialName("image_file") - public data class ImageFile( + public data class Image( /** * The File ID of the image in the message content. */ @@ -66,6 +66,7 @@ public sealed interface MessageContent { ) : MessageContent } + /** * The text content of the message value and annotations. */ diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt index 75b1da6b..4bc8841d 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt @@ -3,7 +3,6 @@ package com.aallam.openai.api.message import com.aallam.openai.api.BetaOpenAI import com.aallam.openai.api.core.Role import com.aallam.openai.api.message.MessageRequestContent.ListContent -import com.aallam.openai.api.message.MessageRequestContent.TextContent import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable @@ -19,7 +18,7 @@ public class MessageRequest( @SerialName("role") public val role: Role, /** - * The content of the message. + * The content of the message. Change this field to support multi-type content. */ @SerialName("content") public val messageContent: MessageRequestContent, @@ -42,7 +41,7 @@ public class MessageRequest( metadata: Map? = null, ) : this( role = role, - messageContent = TextContent(content), + messageContent = MessageRequestContent.TextContent(content), attachments = attachments, metadata = metadata ) @@ -61,7 +60,7 @@ public class MessageRequest( public val content: String get() = when (messageContent) { - is TextContent -> messageContent.content + is MessageRequestContent.TextContent -> messageContent.content else -> error("Content is not text") } } @@ -127,4 +126,3 @@ public class MessageRequestBuilder { ) } } - diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt index 2f8c5a7d..4a9b2fc8 100644 --- a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt +++ b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt @@ -3,9 +3,7 @@ package com.aallam.openai.api.message import com.aallam.openai.api.BetaOpenAI import com.aallam.openai.api.OpenAIDsl import com.aallam.openai.api.file.FileId -import com.aallam.openai.api.message.MessageRequestPart.ImageFilePart -import com.aallam.openai.api.message.MessageRequestPart.ImageUrlPart -import com.aallam.openai.api.message.MessageRequestPart.TextPart +import com.aallam.openai.api.message.MessageRequestPart.* import com.aallam.openai.api.message.internal.MessageRequestContentSerializer import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable