Improve Assistant RunStepDetails and support array content for MessageRequest

openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/ImageFile.kt

@@ -12,5 +12,26 @@ public data class ImageFile(
     /**
      * The File ID of the image in the message content.
      */
-    @SerialName("file_id") val fileId: FileId
+    @SerialName("file_id") val fileId: FileId,
+    /**
+     * Optional Defaults to auto
+     * Specifies the detail level of the image if specified by the user. low uses fewer tokens, you can opt in to high resolution using high.
+     */
+    @SerialName("detail") val detail: String? = null,
 )
+
+/**
+ * Image content part data.
+ */
+@Serializable
+public data class ImageURL(
+    /**
+     * Either a URL of the image or the base64 encoded image data.
+     */
+    @SerialName("url") val url: String,
+
+    /**
+     * Specifies the detail level of the image.
+     */
+    @SerialName("detail") val detail: String? = null,
+)

openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageContent.kt

@@ -35,7 +35,34 @@ public sealed interface MessageContent {
         /**
          * The File ID of the image in the message content.
          */
-        @SerialName("image_file") val imageFile: ImageFile
+        @SerialName("image_file") val imageFile: com.aallam.openai.api.message.ImageFile
+    ) : MessageContent
+
+    /**
+     * References an image File in the content of a message.
+     */
+    @BetaOpenAI
+    @Serializable
+    @SerialName("image_url")
+    public data class ImageURL(
+        /**
+         * The Image Url of the image in the message content.
+         */
+        @SerialName("image_url") val imageUrl: com.aallam.openai.api.message.ImageURL
+    ) : MessageContent
+
+
+    /**
+     * The refusal content generated by the assistant.
+     */
+    @BetaOpenAI
+    @Serializable
+    @SerialName("refusal")
+    public data class Refusal(
+        /**
+         * The data that makes up the refusal.
+         */
+        @SerialName("refusal") val refusal: String
     ) : MessageContent
 
 }

openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequest.kt

@@ -2,6 +2,7 @@ package com.aallam.openai.api.message
 
 import com.aallam.openai.api.BetaOpenAI
 import com.aallam.openai.api.core.Role
+import com.aallam.openai.api.message.MessageRequestContent.ListContent
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
 
@@ -17,9 +18,9 @@ public class MessageRequest(
     @SerialName("role") public val role: Role,
 
     /**
-     * The content of the message.
+     * The content of the message. Change this field to support multi-type content.
      */
-    @SerialName("content") public val content: String,
+    @SerialName("content") public val messageContent: MessageRequestContent,
 
     /**
      * A list of files attached to the message.
@@ -32,7 +33,37 @@ public class MessageRequest(
      * Keys can be a maximum of 64 characters long, and values can be a maximum of 512 characters long.
      */
     @SerialName("metadata") public val metadata: Map<String, String>? = null,
-)
+) {
+    public constructor(
+        role: Role,
+        content: String,
+        attachments: List<Attachment>? = null,
+        metadata: Map<String, String>? = null,
+    ) : this(
+        role = role,
+        messageContent = MessageRequestContent.TextContent(content),
+        attachments = attachments,
+        metadata = metadata
+    )
+
+    public constructor(
+        role: Role,
+        content: List<MessageRequestPart>,
+        attachments: List<Attachment>? = null,
+        metadata: Map<String, String>? = null,
+    ) : this(
+        role = role,
+        messageContent = ListContent(content),
+        attachments = attachments,
+        metadata = metadata
+    )
+
+    public val content: String
+        get() = when (messageContent) {
+            is MessageRequestContent.TextContent -> messageContent.content
+            else -> error("Content is not text")
+        }
+}
 
 /**
  * A message request builder.
@@ -68,10 +99,30 @@ public class MessageRequestBuilder {
      */
     public var metadata: Map<String, String>? = null
 
-    public fun build(): MessageRequest = MessageRequest(
-        role = requireNotNull(role) { "role is required" },
-        content = requireNotNull(content) { "content is required" },
-        attachments = attachments,
-        metadata = metadata
-    )
+    /**
+     * The contents of the assistant request message.
+     */
+    internal val parts = mutableListOf<MessageRequestPart>()
+
+    /**
+     * The contents of the message.
+     */
+    public fun content(block: MessageRequestPartBuilder.() -> Unit) {
+        this.parts += MessageRequestPartBuilder().apply(block).build()
+    }
+
+    /**
+     * Create [MessageRequest] instance.
+     */
+    public fun build(): MessageRequest {
+        require(!(content != null && parts.isNotEmpty())) { "Cannot set both content string and content parts" }
+        val messageContent = content?.let { MessageRequestContent.TextContent(it) }
+            ?: ListContent(parts)
+        return MessageRequest(
+            role = requireNotNull(role) { "role is required " },
+            messageContent = messageContent,
+            attachments = attachments,
+            metadata = metadata
+        )
+    }
 }

openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/MessageRequestContent.kt

@@ -0,0 +1,122 @@
+package com.aallam.openai.api.message
+
+import com.aallam.openai.api.BetaOpenAI
+import com.aallam.openai.api.OpenAIDsl
+import com.aallam.openai.api.file.FileId
+import com.aallam.openai.api.message.MessageRequestPart.*
+import com.aallam.openai.api.message.internal.MessageRequestContentSerializer
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
+import kotlin.jvm.JvmInline
+
+/**
+ * The contents of the assistant request message.
+ */
+@BetaOpenAI
+@Serializable(with = MessageRequestContentSerializer::class)
+public sealed interface MessageRequestContent {
+    /**
+     * The assistant message content as text.
+     */
+    @BetaOpenAI
+    @JvmInline
+    @Serializable
+    public value class TextContent(public val content: String) : MessageRequestContent
+
+    /**
+     *  The assistant message content as a list of content parts.
+     */
+    @BetaOpenAI
+    @JvmInline
+    @Serializable
+    public value class ListContent(public val content: List<MessageRequestPart>) : MessageRequestContent
+}
+
+
+@BetaOpenAI
+@Serializable
+public sealed interface MessageRequestPart {
+    /**
+     * The content of the message as text.
+     */
+    @BetaOpenAI
+    @Serializable
+    @SerialName("text")
+    public data class TextPart(
+        /**
+         * The text content of the message value and annotations.
+         */
+        @SerialName("text") val text: String
+    ) : MessageRequestPart
+
+    /**
+     * References an image File in the content of a message.
+     */
+    @BetaOpenAI
+    @Serializable
+    @SerialName("image_file")
+    public data class ImageFilePart(
+        /**
+         * The Image file object of the image in the message content.
+         */
+        @SerialName("image_file") val imageFile: ImageFile
+    ) : MessageRequestPart
+
+    /**
+     * References an image url in the content of a message.
+     */
+    @BetaOpenAI
+    @Serializable
+    @SerialName("image_url")
+    public data class ImageUrlPart(
+        /**
+         * The Image url object of the image in the message content.
+         */
+        @SerialName("image_url") val imageUrl: ImageURL
+    ) : MessageRequestPart
+
+}
+
+@OptIn(BetaOpenAI::class)
+@OpenAIDsl
+public class MessageRequestPartBuilder {
+
+    private val parts = mutableListOf<MessageRequestPart>()
+
+    /**
+     * Text content part.
+     *
+     * @param text the text content.
+     */
+    public fun text(text: String) {
+        this.parts += TextPart(text)
+    }
+
+    /**
+     * Image file content part.
+     *
+     * @param id the image file.
+     * @param detail the image detail.
+     */
+    public fun imageFile(id: FileId, detail: String? = null) {
+        this.parts += ImageFilePart(ImageFile(id, detail))
+    }
+
+    /**
+     * Image url content part.
+     *
+     * @param url the image url.
+     * @param detail the image detail.
+     */
+    public fun imageUrl(url: String, detail: String? = null) {
+        this.parts += ImageUrlPart(ImageURL(url, detail))
+    }
+
+
+    /**
+     * Create a list of [MessageRequestPart]s.
+     */
+    public fun build(): List<MessageRequestPart> {
+        return parts
+    }
+}

openai-core/src/commonMain/kotlin/com.aallam.openai.api/message/internal/MessageRequestContentSerializer.kt

@@ -0,0 +1,28 @@
+package com.aallam.openai.api.message.internal
+
+
+import com.aallam.openai.api.BetaOpenAI
+import com.aallam.openai.api.message.MessageRequestContent
+import com.aallam.openai.api.message.MessageRequestContent.ListContent
+import com.aallam.openai.api.message.MessageRequestContent.TextContent
+import kotlinx.serialization.DeserializationStrategy
+import kotlinx.serialization.SerializationException
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.JsonContentPolymorphicSerializer
+import kotlinx.serialization.json.JsonElement
+import kotlinx.serialization.json.JsonPrimitive
+
+/**
+ * Serializer for [MessageRequestContent].
+ */
+@OptIn(BetaOpenAI::class)
+internal class MessageRequestContentSerializer :
+    JsonContentPolymorphicSerializer<MessageRequestContent>(MessageRequestContent::class) {
+    override fun selectDeserializer(element: JsonElement): DeserializationStrategy<MessageRequestContent> {
+        return when (element) {
+            is JsonPrimitive -> TextContent.serializer()
+            is JsonArray -> ListContent.serializer()
+            else -> throw SerializationException("Unsupported JSON element: $element")
+        }
+    }
+}