New content format

Sources/OpenAI/Public/Models/ChatQuery.swift

@@ -7,6 +7,8 @@
 
 import Foundation
 
+
+
 // See more https://platform.openai.com/docs/guides/text-generation/json-mode
 public struct ResponseFormat: Codable, Equatable {
     public static let jsonObject = ResponseFormat(type: .jsonObject)
@@ -20,10 +22,89 @@ public struct ResponseFormat: Codable, Equatable {
     }
 }
 
+public struct ChatContent: Codable, Equatable {
+    let type: ChatContentType
+    let value: String
+
+    public enum ChatContentType: String, Codable {
+        case text
+        case imageUrl = "image_url"
+    }
+
+    public struct ImageUrl: Codable, Equatable {
+        let url: String
+
+        enum CodingKeys: CodingKey {
+            case url
+        }
+    }
+
+
+    enum CodingKeys: CodingKey {
+        case type
+        case value
+    }
+
+    public static func text(_ text: String) -> Self {
+        Self.init(text)
+    }
+
+    public static func imageUrl(_ url: String) -> Self {
+        Self.init(type: .imageUrl, value: url)
+    }
+
+    public init(type: ChatContentType, value: String) {
+        self.type = type
+        self.value = value
+    }
+
+    public init(_ text: String) {
+        self.type = .text
+        self.value = text
+    }
+
+    // we need to perform a custom encoding since the `value` key is variable based on the `type`
+    public func encode(to encoder: Encoder) throws {
+        var container = encoder.container(keyedBy: ChatContent.CodingKeys.self)
+        var dynamicContainer = encoder.container(keyedBy: DynamicKey.self)
+
+        try container.encode(type, forKey: .type)
+
+        switch self.type {
+        case .text:
+            try dynamicContainer.encode(value, forKey: .init(stringValue: "text"))
+            break
+        case .imageUrl:
+            var nested = dynamicContainer.nestedContainer(keyedBy: ImageUrl.CodingKeys.self, forKey: .init(stringValue: "image_url"))
+            try nested.encode(value, forKey: .url)
+            break
+        }
+    }
+
+    public init(from decoder: Decoder) throws {
+        let container = try decoder.container(keyedBy: CodingKeys.self)
+        self.type = try container.decode(ChatContentType.self, forKey: .type)
+
+        let dynamicContainer = try decoder.container(keyedBy: DynamicKey.self)
+
+        switch self.type {
+        case .text:
+            self.value = try dynamicContainer.decode(String.self, forKey: .init(stringValue: "text"))
+            break
+        case .imageUrl:
+            let nested = try dynamicContainer.nestedContainer(keyedBy: ImageUrl.CodingKeys.self, forKey: .init(stringValue: "image_url"))
+            self.value = try nested.decode(String.self, forKey: .url)
+            break
+        }
+    }
+}
+
+
 public struct Chat: Codable, Equatable {
     public let role: Role
     /// The contents of the message. `content` is required for all messages except assistant messages with function calls.
-    public let content: String?
+    public let content: StringOrCodable<[ChatContent]>?
+
     /// The name of the author of this message. `name` is required if role is `function`, and it should be the name of the function whose response is in the `content`. May contain a-z, A-Z, 0-9, and underscores, with a maximum length of 64 characters.
     public let name: String?
     public let functionCall: ChatFunctionCall?
@@ -42,9 +123,33 @@ public struct Chat: Codable, Equatable {
         case functionCall = "function_call"
     }
 
-    public init(role: Role, content: String? = nil, name: String? = nil, functionCall: ChatFunctionCall? = nil) {
+    public init(role: Role, content stringContent: String? = nil, name: String? = nil, functionCall: ChatFunctionCall? = nil) {
+        let stringOrCodable: StringOrCodable<[ChatContent]>?;
+
+        if let string = stringContent {
+            stringOrCodable = .string(string)
+        } else {
+            stringOrCodable = nil
+        }
+
+        self.init(role: role, contents: stringOrCodable, name: name, functionCall: functionCall)
+    }
+
+    public init(role: Role, content arr: [ChatContent]? = nil ,name: String? = nil, functionCall: ChatFunctionCall? = nil) {
+        let stringOrCodable: StringOrCodable<[ChatContent]>?
+
+        if let arr = arr {
+            stringOrCodable = .object(arr)
+        } else {
+            stringOrCodable = nil
+        }
+
+        self.init(role: role, contents: stringOrCodable, name: name, functionCall: functionCall)
+    }
+
+    public init(role: Role, contents: StringOrCodable<[ChatContent]>? = nil, name: String? = nil, functionCall: ChatFunctionCall? = nil) {
         self.role = role
-        self.content = content
+        self.content = contents
         self.name = name
         self.functionCall = functionCall
     }

Tests/OpenAITests/OpenAITests.swift

@@ -63,19 +63,20 @@ class OpenAITests: XCTestCase {
     }
 
     func testChats() async throws {
-       let query = ChatQuery(model: .gpt4, messages: [
-           .init(role: .system, content: "You are Librarian-GPT. You know everything about the books."),
-           .init(role: .user, content: "Who wrote Harry Potter?")
-       ])
-       let chatResult = ChatResult(id: "id-12312", object: "foo", created: 100, model: .gpt3_5Turbo, choices: [
-        .init(index: 0, message: .init(role: .system, content: "bar"), finishReason: "baz"),
-        .init(index: 0, message: .init(role: .user, content: "bar1"), finishReason: "baz1"),
-        .init(index: 0, message: .init(role: .assistant, content: "bar2"), finishReason: "baz2")
+        let query = ChatQuery(model: .gpt4, messages: [
+            .init(role: .system, content: "You are Librarian-GPT. You know everything about the books."),
+            .init(role: .user, content: "Who wrote Harry Potter?")
+        ])
+        let chatResult = ChatResult(id: "id-12312", object: "foo", created: 100, model: .gpt3_5Turbo, choices: [
+            .init(index: 0, message: .init(role: .system, content: "bar"), finishReason: "baz"),
+            .init(index: 0, message: .init(role: .user, content: "bar1"), finishReason: "baz1"),
+            .init(index: 0, message: .init(role: .assistant, content: "bar2"), finishReason: "baz2")
         ], usage: .init(promptTokens: 100, completionTokens: 200, totalTokens: 300))
-       try self.stub(result: chatResult)
+        try self.stub(result: chatResult)
+
+        let result = try await openAI.chats(query: query)
 
-       let result = try await openAI.chats(query: query)
-       XCTAssertEqual(result, chatResult)
+        XCTAssertEqual(result, chatResult)
     }
 
     func testChatsFunction() async throws {

Tests/OpenAITests/OpenAITestsDecoder.swift

@@ -200,6 +200,42 @@ class OpenAITestsDecoder: XCTestCase {
 
         XCTAssertEqual(chatQueryAsDict, expectedValueAsDict)
     }
+
+    func testChatContentImage() async throws {
+        let expectedValue = """
+        {
+            "type": "image_url",
+            "image_url": {
+                "url": "https://example.com"
+            }
+        }
+        """
+
+        let data = ChatContent(type: .imageUrl, value: "https://example.com")
+
+        // To compare serialized JSONs we first convert them both into NSDictionary which are comparable (unline native swift dictionaries)
+        let resultDict = try jsonDataAsNSDictionary(JSONEncoder().encode(data))
+        let expectedValueAsDict = try jsonDataAsNSDictionary(expectedValue.data(using: .utf8)!)
+
+        XCTAssertEqual(resultDict, expectedValueAsDict)
+    }
+
+    func testChatContentText() async throws {
+        let expectedValue = """
+        {
+            "type": "text",
+            "text": "hello world"
+        }
+        """
+
+        let data = ChatContent(type: .text, value: "hello world")
+
+        // To compare serialized JSONs we first convert them both into NSDictionary which are comparable (unline native swift dictionaries)
+        let resultDict = try jsonDataAsNSDictionary(JSONEncoder().encode(data))
+        let expectedValueAsDict = try jsonDataAsNSDictionary(expectedValue.data(using: .utf8)!)
+
+        XCTAssertEqual(resultDict, expectedValueAsDict)
+    }
 
     func testChatCompletionWithFunctionCall() async throws {
         let data = """
@@ -235,12 +271,16 @@ class OpenAITestsDecoder: XCTestCase {
             created: 1677652288,
             model: .gpt3_5Turbo,
             choices: [
-                .init(index: 0, message:
-                        Chat(role: .assistant,
-                             functionCall: ChatFunctionCall(name: "get_current_weather", arguments: nil)),
-                      finishReason: "function_call")
+                .init(
+                    index: 0,
+                    // TODO: Fix "Ambiguous use of 'init(role:content:name:functionCall:)'" when omitting content
+                    // maybe by removing `init` the overload?
+                    message: Chat(role: .assistant, content: nil as String?, functionCall: .init(name: "get_current_weather", arguments: nil)),
+                    finishReason: "function_call"
+                )
             ],
             usage: .init(promptTokens: 82, completionTokens: 18, totalTokens: 100))
+
         try decode(data, expectedValue)
     }