From 8e75310a779ce4d3e2f8f214a6f57dcd20f65cc0 Mon Sep 17 00:00:00 2001 From: Tomas Cayuelas Date: Wed, 11 Oct 2023 16:12:58 +0200 Subject: [PATCH 1/3] Fix problem with EOF using aallam.openai client --- .../conversation/llm/openai/Conversions.kt | 4 ++ .../xef/conversation/llm/openai/OpenAI.kt | 45 ++++++++++++++----- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/Conversions.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/Conversions.kt index 3ec752780..2795c2f64 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/Conversions.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/Conversions.kt @@ -5,6 +5,7 @@ import com.aallam.openai.api.chat.ChatCompletionChunk as OAIChatCompletionChunk import com.aallam.openai.api.chat.ChatDelta as OAIChatDelta import com.aallam.openai.api.chat.ChatRole import com.aallam.openai.api.core.Usage as OAIUsage +import com.aallam.openai.api.http.Timeout as OAITimeout import com.xebia.functional.xef.llm.models.chat.ChatChunk import com.xebia.functional.xef.llm.models.chat.ChatCompletionChunk import com.xebia.functional.xef.llm.models.chat.ChatDelta @@ -61,3 +62,6 @@ internal fun OAIChatDelta.toInternal() = else FunctionCall(it.name, it.arguments) } ) + +internal fun OpenAI.Timeout.toOAITimeout() = + OAITimeout(this.request, this.connect, this.socket) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index ce34ca426..60d92faef 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -22,12 +22,32 @@ import kotlin.jvm.JvmField import kotlin.jvm.JvmOverloads import kotlin.jvm.JvmStatic import kotlin.jvm.JvmSynthetic +import kotlin.time.Duration +import kotlin.time.Duration.Companion.minutes +import kotlin.time.Duration.Companion.seconds private const val KEY_ENV_VAR = "OPENAI_TOKEN" private const val HOST_ENV_VAR = "OPENAI_HOST" -class OpenAI(internal var token: String? = null, internal var host: String? = null) : - AutoCloseable, AutoClose by autoClose() { +class OpenAI( + internal var token: String? = null, + internal var host: String? = null, + internal var timeout: Timeout = Timeout.default() +) : AutoCloseable, AutoClose by autoClose() { + + class Timeout private constructor( + val request: Duration, + val connect: Duration, + val socket: Duration, + ) { + companion object { + private val REQUEST_TIMEOUT = 30.seconds + private val CONNECT_TIMEOUT = 10.minutes + private val SOCKET_TIMEOUT = 10.minutes + + fun default(): Timeout = Timeout(REQUEST_TIMEOUT, CONNECT_TIMEOUT, SOCKET_TIMEOUT) + } + } private fun openAITokenFromEnv(): String { return getenv(KEY_ENV_VAR) @@ -65,6 +85,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu token = getToken(), logging = LoggingConfig(LogLevel.None), headers = mapOf("Authorization" to " Bearer ${getToken()}"), + timeout = this.timeout.toOAITimeout() ) .let { autoClose(it) } @@ -132,7 +153,16 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu DALLE_2, ) - suspend fun findModel(modelId: String): Any? { // TODO: impl of abstract provider function + suspend fun spawnModel( + modelId: String, + baseModel: T + ): T { // TODO: impl of abstract provider function + if (findModel(modelId) == null) error("model not found") + return baseModel.copy(ModelType.FineTunedModel(modelId, baseModel = baseModel.modelType)) as? T + ?: error("${baseModel::class} does not follow contract to return the most specific type") + } + + private suspend fun findModel(modelId: String): Any? { // TODO: impl of abstract provider function val model = try { defaultClient.model(ModelId(modelId)) @@ -145,15 +175,6 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu return ModelType.TODO(model.id.id) } - suspend fun spawnModel( - modelId: String, - baseModel: T - ): T { // TODO: impl of abstract provider function - if (findModel(modelId) == null) error("model not found") - return baseModel.copy(ModelType.FineTunedModel(modelId, baseModel = baseModel.modelType)) as? T - ?: error("${baseModel::class} does not follow contract to return the most specific type") - } - companion object { @JvmField val FromEnvironment: OpenAI = OpenAI() From 8ee39ac86a4b23f50e8d09b45c41daf2e6b86f75 Mon Sep 17 00:00:00 2001 From: Tomas Cayuelas Date: Wed, 11 Oct 2023 17:06:41 +0200 Subject: [PATCH 2/3] Update request_timeout to 60 seconds by default --- .../com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 60d92faef..d290d4dab 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -41,7 +41,7 @@ class OpenAI( val socket: Duration, ) { companion object { - private val REQUEST_TIMEOUT = 30.seconds + private val REQUEST_TIMEOUT = 60.seconds private val CONNECT_TIMEOUT = 10.minutes private val SOCKET_TIMEOUT = 10.minutes From 7bc9da560432ffbc2eaa4fca1e332f77756d05a8 Mon Sep 17 00:00:00 2001 From: Tomas Cayuelas Date: Wed, 11 Oct 2023 19:02:30 +0200 Subject: [PATCH 3/3] Allows the user to build Timeout depending on the use case --- .../com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index d290d4dab..dd6a0713e 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -35,7 +35,7 @@ class OpenAI( internal var timeout: Timeout = Timeout.default() ) : AutoCloseable, AutoClose by autoClose() { - class Timeout private constructor( + class Timeout( val request: Duration, val connect: Duration, val socket: Duration,