feat(chat): add "vision" feature (#258)

aallam · Nov 10, 2023 · a19b649 · a19b649
1 parent 04695f2
commit a19b649
Show file tree

Hide file tree

Showing 14 changed files with 687 additions and 19 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,7 +1,9 @@
-# Unreleased
+## Unreleased
 
 ### Added
-- **Images**: Support for model selection for `ImageCreation`, `ImageEdit` and `ImageVariations`
+- **Images**: Support for model selection for `ImageCreation`, `ImageEdit` and `ImageVariations` (#257)
+- **Chat**: add tool calls (#256)
+- **Chat**: add vision feature (#258)
 
 # 3.5.1
 > Published 05 Nov 2023

diff --git a/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestChatCompletions.kt b/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestChatCompletions.kt
@@ -129,4 +129,24 @@ class TestChatCompletions : TestOpenAI() {
         assertNotNull(answer.question)
         assertNotNull(answer.response)
     }
+
+    @Test
+    fun vision() = test {
+        val request = chatCompletionRequest {
+            model = ModelId("gpt-4-vision-preview")
+            messages {
+                user {
+                    content {
+                        text("What are in these images? Is there any difference between them?")
+                        image("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")
+                        image("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")
+                    }
+                }
+            }
+            maxTokens = 300
+        }
+        val response = openAI.chatCompletion(request)
+        val content = response.choices.first().message.content.orEmpty()
+        assertNotNull(content)
+    }
 }
diff --git a/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestChatVision.kt b/openai-client/src/commonTest/kotlin/com/aallam/openai/client/TestChatVision.kt
@@ -0,0 +1,55 @@
+package com.aallam.openai.client
+
+import com.aallam.openai.api.chat.*
+import com.aallam.openai.api.chat.internal.ToolType
+import com.aallam.openai.api.model.ModelId
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import kotlin.test.*
+
+class TestChatVision : TestOpenAI() {
+
+    @Test
+    fun textimage() = test {
+        val request = chatCompletionRequest {
+            model = ModelId("gpt-4-vision-preview")
+            messages {
+                user {
+                    content {
+                        text("What’s in this image?")
+                        image("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")
+                    }
+                }
+            }
+            maxTokens = 300
+        }
+        val response = openAI.chatCompletion(request)
+        val content = response.choices.first().message.content.orEmpty()
+        assertNotNull(content)
+    }
+
+    @Test
+    fun multiImage() = test {
+        val request = chatCompletionRequest {
+            model = ModelId("gpt-4-vision-preview")
+            messages {
+                user {
+                    content {
+                        text("What are in these images? Is there any difference between them?")
+                        image("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")
+                        image("https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg")
+                    }
+                }
+            }
+            maxTokens = 300
+        }
+        val response = openAI.chatCompletion(request)
+        val content = response.choices.first().message.content.orEmpty()
+        assertNotNull(content)
+    }
+}
diff --git a/openai-client/src/jvmTest/kotlin/com/aallam/openai/client/TestChatVisionJVM.kt b/openai-client/src/jvmTest/kotlin/com/aallam/openai/client/TestChatVisionJVM.kt
@@ -0,0 +1,43 @@
+package com.aallam.openai.client
+
+import com.aallam.openai.api.chat.*
+import com.aallam.openai.api.chat.internal.ToolType
+import com.aallam.openai.api.model.ModelId
+import com.aallam.openai.client.TestOpenAI
+import kotlinx.coroutines.flow.launchIn
+import kotlinx.coroutines.flow.onEach
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import okio.FileSystem
+import okio.Okio
+import okio.Path.Companion.toPath
+import kotlin.test.*
+
+class TestChatVisionJVM : TestOpenAI() {
+
+    @Test
+    fun encoded() = test {
+        val byteString = FileSystem.RESOURCES.read("nature.jpeg".toPath()) {
+            readByteString()
+        }
+        val encoded = byteString.base64()
+        val request = chatCompletionRequest {
+            model = ModelId("gpt-4-vision-preview")
+            messages {
+                user {
+                    content {
+                        text("What’s in this image?")
+                        image("data:image/jpeg;base64,$encoded")
+                    }
+                }
+            }
+            maxTokens = 300
+        }
+        val response = openAI.chatCompletion(request)
+        val content = response.choices.first().message.content.orEmpty()
+        assertNotNull(content)
+    }
+}
diff --git a/openai-client/src/jvmTest/resources/nature.jpeg b/openai-client/src/jvmTest/resources/nature.jpeg
diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/chat/ChatChoice.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/chat/ChatChoice.kt
@@ -24,5 +24,5 @@ public data class ChatChoice(
     /**
      * The reason why OpenAI stopped generating.
      */
-    @SerialName("finish_reason") public val finishReason: FinishReason,
+    @SerialName("finish_reason") public val finishReason: FinishReason? = null,
 )
diff --git a/openai-core/src/commonMain/kotlin/com.aallam.openai.api/chat/ChatCompletionRequest.kt b/openai-core/src/commonMain/kotlin/com.aallam.openai.api/chat/ChatCompletionRequest.kt
@@ -338,6 +338,34 @@ public class ChatMessagesBuilder {
     public fun message(block: ChatMessageBuilder.() -> Unit) {
         messages += ChatMessageBuilder().apply(block).build()
     }
+
+    /**
+     * System chat message.
+     */
+    public fun system(block: SystemMessageBuilder.() -> Unit) {
+        this.messages += SystemMessageBuilder().apply(block).build()
+    }
+
+    /**
+     * User chat message.
+     */
+    public fun user(block: UserMessageBuilder.() -> Unit) {
+        this.messages += UserMessageBuilder().apply(block).build()
+    }
+
+    /**
+     * Assistant chat message.
+     */
+    public fun assistant(block: AssistantMessageBuilder.() -> Unit) {
+        this.messages += AssistantMessageBuilder().apply(block).build()
+    }
+
+    /**
+     * Tool chat message.
+     */
+    public fun tool(block: ToolMessageBuilder.() -> Unit) {
+        this.messages += ToolMessageBuilder().apply(block).build()
+    }
 }
 
 /**