image support

Author: yukiny0811

Sources/OpenAIStructure/OpenAIRequest.swift

@@ -63,4 +63,80 @@ public enum OpenAIRequest {
         let result = try JSONDecoder().decode(T.self, from: text.data(using: .utf8)!)
         return result
     }
+
+    struct PayloadWithImage<Schema: Codable>: Codable {
+        let model: String
+        let input: [Input]
+        let reasoning: Reasoning
+        let instructions: String
+        let text: TextObject
+        struct Reasoning: Codable {
+            let effort: String?
+        }
+        struct TextObject: Codable {
+            let format: Schema
+        }
+        struct Input: Codable {
+            var role: String
+            var content: [Content]
+            init(content: [Content]) {
+                self.role = "user"
+                self.content = content
+            }
+            struct Content: Codable {
+                var type: String
+                var detail: String?
+                var image_url: String?
+                var text: String?
+            }
+        }
+    }
+
+    public static func request<T: OpenAIStructureObject>(input: String, instructions: String, image: Data, model: OpenAIModel, object: T.Type, apiKey: String) async throws -> T {
+        var req = URLRequest(url: endpoint)
+        req.httpMethod = "POST"
+        req.addValue("application/json", forHTTPHeaderField: "Content-Type")
+        req.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization")
+        let payload: PayloadWithImage<SchemaRequest>
+        if model.hasReasoning, case let OpenAIModel.o4_mini(reasoningEffort) = model {
+            payload = PayloadWithImage(
+                model: model.modelName,
+                input: [
+                    .init(
+                        content: [
+                            .init(type: "input_text", text: input),
+                            .init(type: "input_image", detail: "high", image_url: "data:image/jpeg;base64," + image.base64EncodedString())
+                        ]
+                    )
+                ],
+                reasoning: .init(effort: reasoningEffort.rawValue),
+                instructions: instructions,
+                text: .init(format: T.schema)
+            )
+        } else {
+            payload = PayloadWithImage(
+                model: model.modelName,
+                input: [
+                    .init(
+                        content: [
+                            .init(type: "input_text", text: input),
+                            .init(type: "input_image", detail: "high", image_url: "data:image/jpeg;base64," + image.base64EncodedString())
+                        ]
+                    )
+                ],
+                reasoning: .init(effort: nil),
+                instructions: instructions,
+                text: .init(format: T.schema)
+            )
+        }
+        req.httpBody = try JSONEncoder().encode(payload)
+        req.timeoutInterval = 300
+        let (data, _) = try await URLSession.shared.data(for: req)
+        guard let decoded = try? JSONDecoder().decode(OpenAIResponse.self, from: data) else {
+            throw OpenAIStructureError(message: String(data: data, encoding: .utf8) ?? "error")
+        }
+        let text = decoded.output.last!.content!.last!.text!
+        let result = try JSONDecoder().decode(T.self, from: text.data(using: .utf8)!)
+        return result
+    }
 }

Sources/OpenAIStructureClient/Media.xcassets/Contents.json

@@ -0,0 +1,6 @@
+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

Sources/OpenAIStructureClient/Media.xcassets/apple.imageset/Contents.json

@@ -0,0 +1,21 @@
+{
+  "images" : [
+    {
+      "filename" : "apple.png",
+      "idiom" : "universal",
+      "scale" : "1x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "2x"
+    },
+    {
+      "idiom" : "universal",
+      "scale" : "3x"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

Sources/OpenAIStructureClient/Media.xcassets/apple.imageset/apple.png

@@ -1,5 +1,6 @@
 import OpenAIStructure
 import Foundation
+import AppKit
 
 @OpenAIStructure(name: "talk_theme_pack")
 struct TalkThemePack {
@@ -51,10 +52,35 @@ do {
         instructions: "Provide some talk themes.",
         model: .gpt4_1_nano,
         object: TalkThemePack.self,
-        apiKey: "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"
+        apiKey: ""
     )
     print(result)
 } catch {
     print(error)
 }
 
+print("-------image-------")
+
+@OpenAIStructure(name: "describe_image")
+struct ImageDescriber {
+
+    @Field("description of the image")
+    var image_description: String
+}
+
+do {
+    let cgImage = Bundle.module.image(forResource: "apple")!.cgImage(forProposedRect: nil, context: nil, hints: nil)!
+    let rep = NSBitmapImageRep(cgImage: cgImage)
+    let imageData = rep.representation(using: .png, properties: [:])!
+    let result = try await OpenAIRequest.request(
+        input: "Talk theme related to food.",
+        instructions: "Provide some talk themes.",
+        image: imageData,
+        model: .gpt4_1_nano,
+        object: ImageDescriber.self,
+        apiKey: ""
+    )
+    print(result)
+} catch {
+    print(error)
+}