From ebb0883a5c4ce77cc5e4896d12e6f7ea0af2a825 Mon Sep 17 00:00:00 2001 From: ron Date: Tue, 3 Oct 2023 16:01:59 +0200 Subject: [PATCH 01/19] cherrypicked most changes from former branch --- .../functional/xef/conversation/AutoClose.kt | 2 + .../com/xebia/functional/xef/llm/LLM.kt | 23 +-- .../xef/llm/models/MaxContextLength.kt | 9 ++ .../functional/xef/llm/models/ModelID.kt | 5 + .../com/xebia/functional/gpt4all/GPT4All.kt | 19 +-- .../gpt4all/HuggingFaceLocalEmbeddings.kt | 5 +- .../com/xebia/functional/xef/gcp/GCP.kt | 8 +- .../functional/xef/gcp/models/GcpChat.kt | 14 +- .../xef/conversation/llm/openai/OpenAI.kt | 80 ++++++++-- .../llm/openai/models/OpenAIModel.kt | 27 ++++ .../xebia/functional/tokenizer/ModelType.kt | 140 ++++++------------ 11 files changed, 179 insertions(+), 153 deletions(-) create mode 100644 core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt create mode 100644 core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/ModelID.kt create mode 100644 openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/AutoClose.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/AutoClose.kt index ed76771c6..a157bfc8a 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/AutoClose.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/conversation/AutoClose.kt @@ -10,6 +10,8 @@ import io.ktor.utils.io.core.* */ interface AutoClose : AutoCloseable { fun autoClose(autoCloseable: A): A + + fun A.autoCloseBind(): A = autoClose(this) } /** DSL method to use AutoClose */ diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 4e26923dd..628a4a6b2 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -1,31 +1,22 @@ package com.xebia.functional.xef.llm import com.xebia.functional.tokenizer.Encoding -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.Message -sealed interface LLM : AutoCloseable { +// sealed modifier temporarily removed as OAI's implementation of tokensFromMessages has to extend and override LLM +/*sealed */interface LLM : AutoCloseable { - val modelType: ModelType + val modelID: ModelID - @Deprecated("use modelType.name instead", replaceWith = ReplaceWith("modelType.name")) + @Deprecated("use modelID.value instead", replaceWith = ReplaceWith("modelID.value")) val name - get() = modelType.name + get() = modelID.value fun tokensFromMessages( messages: List ): Int { // TODO: naive implementation with magic numbers - fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = - messages.sumOf { message -> - countTokens(message.role.name) + - countTokens(message.content) + - tokensPerMessage + - tokensPerName - } + 3 - return modelType.encoding.countTokensFromMessages( - tokensPerMessage = modelType.tokensPerMessage, - tokensPerName = modelType.tokensPerName - ) + modelType.tokenPadding + TODO() } override fun close() = Unit diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt new file mode 100644 index 000000000..bb32e416f --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt @@ -0,0 +1,9 @@ +package com.xebia.functional.xef.llm.models + +sealed interface MaxContextLength { + /** one total length of input and output combined */ + data class Combined(val total: Int) : MaxContextLength + + /** two separate max lengths for input and output respectively */ + data class FixIO(val input: Int, val output: Int) : MaxContextLength +} diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/ModelID.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/ModelID.kt new file mode 100644 index 000000000..6051eab6c --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/ModelID.kt @@ -0,0 +1,5 @@ +package com.xebia.functional.xef.llm.models + +import kotlin.jvm.JvmInline + +@JvmInline value class ModelID(val value: String) diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt index 8462c9126..ce5522dfb 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt @@ -4,11 +4,11 @@ import ai.djl.training.util.DownloadUtils import ai.djl.training.util.ProgressBar import com.hexadevlabs.gpt4all.LLModel import com.xebia.functional.tokenizer.EncodingType -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.conversation.PlatformConversation import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Completion +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest @@ -64,8 +64,17 @@ interface GPT4All : AutoCloseable, Chat, Completion { } } + override val modelID = ModelID(path.name) val llModel = LLModel(path) + override suspend fun estimateTokens(message: String): Int { + TODO("Not yet implemented") + } + + override suspend fun estimateTokens(messages: List): Int { + return 0 + } + override suspend fun createCompletion(request: CompletionRequest): CompletionResult = with(request) { val config = LLModel.config() @@ -155,16 +164,8 @@ interface GPT4All : AutoCloseable, Chat, Completion { ) } - override fun tokensFromMessages(messages: List): Int { - return 0 - } - - override val name: String = path.name - override fun close(): Unit = llModel.close() - override val modelType: ModelType = ModelType.LocalModel(name, EncodingType.CL100K_BASE, 4096) - private fun List.buildPrompt(): String { val messages: String = joinToString("") { message -> when (message.role) { diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt index a7a7a7ab4..8d7f84651 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt @@ -1,7 +1,6 @@ package com.xebia.functional.gpt4all import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.llm.Embeddings import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest @@ -10,11 +9,11 @@ import com.xebia.functional.xef.llm.models.embeddings.RequestConfig import com.xebia.functional.xef.llm.models.usage.Usage class HuggingFaceLocalEmbeddings( - override val modelType: ModelType, + override val modelID: com.xebia.functional.xef.llm.models.ModelID, artifact: String, ) : Embeddings { - private val tokenizer = HuggingFaceTokenizer.newInstance("${modelType.name}/$artifact") + private val tokenizer = HuggingFaceTokenizer.newInstance("${modelID.value}/$artifact") override val name: String = HuggingFaceLocalEmbeddings::class.java.canonicalName diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt index 5b5934ef6..5f7d2c73d 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt @@ -1,7 +1,6 @@ package com.xebia.functional.xef.gcp import arrow.core.nonEmptyListOf -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.conversation.PlatformConversation @@ -9,6 +8,7 @@ import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.gcp.models.GcpChat import com.xebia.functional.xef.gcp.models.GcpEmbeddings import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore import kotlin.jvm.JvmField @@ -47,10 +47,8 @@ class GCP(projectId: String? = null, location: VertexAIRegion? = null, token: St val defaultClient = GcpClient(config) - val CODECHAT by lazy { GcpChat(ModelType.TODO("codechat-bison@001"), defaultClient) } - val TEXT_EMBEDDING_GECKO by lazy { - GcpEmbeddings(ModelType.TODO("textembedding-gecko"), defaultClient) - } + val CODECHAT by lazy { GcpChat(ModelID("codechat-bison@001"), defaultClient) } + val TEXT_EMBEDDING_GECKO by lazy { GcpEmbeddings(ModelID("textembedding-gecko"), defaultClient) } @JvmField val DEFAULT_CHAT = CODECHAT @JvmField val DEFAULT_EMBEDDING = TEXT_EMBEDDING_GECKO diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt index a77746db2..3a26068d6 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt @@ -1,8 +1,8 @@ package com.xebia.functional.xef.gcp.models -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.gcp.GcpClient import com.xebia.functional.xef.llm.Chat +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.usage.Usage import io.ktor.client.* @@ -13,7 +13,7 @@ import kotlinx.uuid.UUID import kotlinx.uuid.generateUUID class GcpChat( - override val modelType: ModelType, + override val modelID: ModelID, private val client: GcpClient, ) : Chat { @@ -23,7 +23,7 @@ class GcpChat( val prompt: String = request.messages.buildPrompt() val response: String = client.promptMessage( - modelType.name, + modelID.value, prompt, temperature = request.temperature, maxOutputTokens = request.maxTokens, @@ -31,9 +31,9 @@ class GcpChat( ) return ChatCompletionResponse( UUID.generateUUID().toString(), - modelType.name, + modelID.value, getTimeMillis().toInt(), - modelType.name, + modelID.value, Usage.ZERO, // TODO: token usage - no information about usage provided by GCP listOf(Choice(Message(Role.ASSISTANT, response, Role.ASSISTANT.name), null, 0)), ) @@ -48,7 +48,7 @@ class GcpChat( val prompt: String = messages.buildPrompt() val response = client.promptMessage( - modelType.name, + modelID.value, prompt, temperature = request.temperature, maxOutputTokens = request.maxTokens, @@ -58,7 +58,7 @@ class GcpChat( ChatCompletionChunk( UUID.generateUUID().toString(), getTimeMillis().toInt(), - modelType.name, + modelID.value, listOf(ChatChunk(delta = ChatDelta(Role.ASSISTANT, response))), Usage .ZERO, // TODO: token usage - no information about usage provided by GCP for codechat diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 17fd7311f..72e2bd99c 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -5,7 +5,7 @@ import com.aallam.openai.api.logging.LogLevel import com.aallam.openai.client.LoggingConfig import com.aallam.openai.client.OpenAI as OpenAIClient import com.aallam.openai.client.OpenAIHost -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AutoClose import com.xebia.functional.xef.conversation.Conversation @@ -14,6 +14,8 @@ import com.xebia.functional.xef.conversation.autoClose import com.xebia.functional.xef.conversation.llm.openai.models.* import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore import kotlin.jvm.JvmField @@ -28,6 +30,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu AutoCloseable, AutoClose by autoClose() { private fun openAITokenFromEnv(): String { + OpenAIHost.OpenAI.baseUrl return getenv(KEY_ENV_VAR) ?: throw AIError.Env.OpenAI(nonEmptyListOf("missing $KEY_ENV_VAR env var")) } @@ -66,51 +69,98 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu ) .let { autoClose(it) } - val GPT_4 by lazy { autoClose(OpenAIChat(ModelType.GPT_4, defaultClient)) } + val GPT_4 by lazy { + OpenAIChat( + ModelID("gpt-4"), + defaultClient, + MaxContextLength.Combined(8192), + EncodingType.CL100K_BASE + ) + .autoCloseBind() + } val GPT_4_0314 by lazy { - autoClose(OpenAIFunChat(ModelType.GPT_4_0314, defaultClient)) // legacy + OpenAIFunChat(ModelID("gpt-4-0314"), defaultClient, EncodingType.CL100K_BASE) + .autoCloseBind() // legacy } - val GPT_4_32K by lazy { autoClose(OpenAIChat(ModelType.GPT_4_32K, defaultClient)) } + val GPT_4_32K by lazy { + OpenAIChat( + ModelID("gpt-4-32k"), + defaultClient, + MaxContextLength.Combined(32768), + EncodingType.CL100K_BASE + ) + .autoCloseBind() + } - val GPT_3_5_TURBO by lazy { autoClose(OpenAIChat(ModelType.GPT_3_5_TURBO, defaultClient)) } + val GPT_3_5_TURBO by lazy { + OpenAIChat( + ModelID("gpt-3.5-turbo"), + defaultClient, + MaxContextLength.Combined(4097), + EncodingType.CL100K_BASE + ) + .autoCloseBind() + } val GPT_3_5_TURBO_16K by lazy { - autoClose(OpenAIChat(ModelType.GPT_3_5_TURBO_16_K, defaultClient)) + OpenAIChat( + ModelID("gpt-3.5-turbo-16k"), + defaultClient, + MaxContextLength.Combined(4097 * 4), + EncodingType.CL100K_BASE + ) + .autoCloseBind() } val GPT_3_5_TURBO_FUNCTIONS by lazy { - autoClose(OpenAIFunChat(ModelType.GPT_3_5_TURBO_FUNCTIONS, defaultClient)) + OpenAIFunChat(ModelID("gpt-3.5-turbo-0613"), defaultClient, EncodingType.CL100K_BASE) + .autoCloseBind() } val GPT_3_5_TURBO_0301 by lazy { - autoClose(OpenAIChat(ModelType.GPT_3_5_TURBO, defaultClient)) // legacy + OpenAIChat( + ModelID("gpt-3.5-turbo-0301"), + defaultClient, + MaxContextLength.Combined(4097), + EncodingType.CL100K_BASE + ) + .autoCloseBind() // legacy } val TEXT_DAVINCI_003 by lazy { - autoClose(OpenAICompletion(ModelType.TEXT_DAVINCI_003, defaultClient)) + OpenAICompletion(ModelID("text-davinci-003"), defaultClient, EncodingType.P50K_BASE) + .autoCloseBind() } val TEXT_DAVINCI_002 by lazy { - autoClose(OpenAICompletion(ModelType.TEXT_DAVINCI_002, defaultClient)) + OpenAICompletion(ModelID("text-davinci-002"), defaultClient, EncodingType.P50K_BASE) + .autoCloseBind() } val TEXT_CURIE_001 by lazy { - autoClose(OpenAICompletion(ModelType.TEXT_SIMILARITY_CURIE_001, defaultClient)) + OpenAICompletion(ModelID("text-similarity-curie-001"), defaultClient, EncodingType.P50K_BASE) + .autoCloseBind() } val TEXT_BABBAGE_001 by lazy { - autoClose(OpenAICompletion(ModelType.TEXT_BABBAGE_001, defaultClient)) + OpenAICompletion(ModelID("text-babbage-001"), defaultClient, EncodingType.P50K_BASE) + .autoCloseBind() } - val TEXT_ADA_001 by lazy { autoClose(OpenAICompletion(ModelType.TEXT_ADA_001, defaultClient)) } + val TEXT_ADA_001 by lazy { + OpenAICompletion(ModelID("text-ada-001"), defaultClient, EncodingType.P50K_BASE).autoCloseBind() + } val TEXT_EMBEDDING_ADA_002 by lazy { - autoClose(OpenAIEmbeddings(ModelType.TEXT_EMBEDDING_ADA_002, defaultClient)) + OpenAIEmbeddings(ModelID("text-embedding-ada-002"), defaultClient, EncodingType.CL100K_BASE) + .autoCloseBind() } - val DALLE_2 by lazy { autoClose(OpenAIImages(ModelType.GPT_3_5_TURBO, defaultClient)) } + val DALLE_2 by lazy { + OpenAIImages(ModelID("dalle-2"), defaultClient, EncodingType.CL100K_BASE).autoCloseBind() + } @JvmField val DEFAULT_CHAT = GPT_3_5_TURBO_16K diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt new file mode 100644 index 000000000..693fdd334 --- /dev/null +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt @@ -0,0 +1,27 @@ +package com.xebia.functional.xef.conversation.llm.openai.models + +import com.xebia.functional.tokenizer.Encoding +import com.xebia.functional.tokenizer.EncodingType +import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.models.chat.Message + +interface OpenAIModel : LLM { + + val encodingType: EncodingType + val encoding + get() = encodingType.encoding + + override fun tokensFromMessages(messages: List): Int { // intermediary solution + fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = + messages.sumOf { message -> + countTokens(message.role.name) + + countTokens(message.content) + + tokensPerMessage + + tokensPerName + } + 3 + return encoding.countTokensFromMessages( + tokensPerMessage = 5, + tokensPerName = 5 + ) + 10 + } +} diff --git a/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt b/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt index e69847507..726059d09 100644 --- a/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt +++ b/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt @@ -1,98 +1,42 @@ -package com.xebia.functional.tokenizer - -import com.xebia.functional.tokenizer.EncodingType.CL100K_BASE -import com.xebia.functional.tokenizer.EncodingType.P50K_BASE -import com.xebia.functional.tokenizer.EncodingType.R50K_BASE -import kotlin.jvm.JvmStatic - -/** - * Formal description of a model and it's properties - * without any capabilities. - */ -sealed class ModelType( - /** - * Returns the name of the model type as used by the OpenAI API. - * - * @return the name of the model type - */ - open val name: String, - open val encodingType: EncodingType, - /** - * Returns the maximum context length that is supported by this model type. Note that - * the maximum context length consists of the amount of prompt tokens and the amount of - * completion tokens (where applicable). - * - * @return the maximum context length for this model type - */ - open val maxContextLength: Int, - - open val tokensPerMessage: Int = 0, - open val tokensPerName: Int = 0, - open val tokenPadding: Int = 20, -) { - - companion object { - @JvmStatic - val DEFAULT_SPLITTER_MODEL = GPT_3_5_TURBO - } - - data class LocalModel(override val name: String, override val encodingType: EncodingType, override val maxContextLength: Int) : ModelType(name, encodingType, maxContextLength) - // chat - object GPT_4 : ModelType("gpt-4", CL100K_BASE, 8192, tokensPerMessage = 3, tokensPerName = 2, tokenPadding = 5) - - object GPT_4_0314 : ModelType("gpt-4-0314", CL100K_BASE, 8192, tokensPerMessage = 3, tokensPerName = 2, tokenPadding = 5) - object GPT_4_32K : ModelType("gpt-4-32k", CL100K_BASE, 32768, tokensPerMessage = 3, tokensPerName = 2, tokenPadding = 5) - object GPT_3_5_TURBO : ModelType("gpt-3.5-turbo", CL100K_BASE, 4097, tokensPerMessage = 4, tokensPerName = 0, tokenPadding = 5) - object GPT_3_5_TURBO_16_K : ModelType("gpt-3.5-turbo-16k", CL100K_BASE, 4097 * 4) - object GPT_3_5_TURBO_FUNCTIONS : ModelType("gpt-3.5-turbo-0613", CL100K_BASE, 4097, tokensPerMessage = 4, tokensPerName = 0, tokenPadding = 200) - // text - object TEXT_DAVINCI_003 : ModelType("text-davinci-003", P50K_BASE, 4097) - object TEXT_DAVINCI_002 : ModelType("text-davinci-002", P50K_BASE, 4097) - object TEXT_DAVINCI_001 : ModelType("text-davinci-001", R50K_BASE, 2049) - object TEXT_CURIE_001 : ModelType("text-curie-001", R50K_BASE, 2049) - object TEXT_BABBAGE_001 : ModelType("text-babbage-001", R50K_BASE, 2049) - object TEXT_ADA_001 : ModelType("text-ada-001", R50K_BASE, 2049) - object DAVINCI : ModelType("davinci", R50K_BASE, 2049) - object CURIE : ModelType("curie", R50K_BASE, 2049) - object BABBAGE : ModelType("babbage", R50K_BASE, 2049) - object ADA : ModelType("ada", R50K_BASE, 2049) - - // code - object CODE_DAVINCI_002 : ModelType("code-davinci-002", P50K_BASE, 8001) - object CODE_DAVINCI_001 : ModelType("code-davinci-001", P50K_BASE, 8001) - object CODE_CUSHMAN_002 : ModelType("code-cushman-002", P50K_BASE, 2048) - object CODE_CUSHMAN_001 : ModelType("code-cushman-001", P50K_BASE, 2048) - object DAVINCI_CODEX : ModelType("davinci-codex", P50K_BASE, 4096) - object CUSHMAN_CODEX : ModelType("cushman-codex", P50K_BASE, 2048) - - // edit - object TEXT_DAVINCI_EDIT_001 : ModelType("text-davinci-edit-001", EncodingType.P50K_EDIT, 3000) - object CODE_DAVINCI_EDIT_001 : ModelType("code-davinci-edit-001", EncodingType.P50K_EDIT, 3000) - - // embeddings - object TEXT_EMBEDDING_ADA_002 : ModelType("text-embedding-ada-002", CL100K_BASE, 8191) - - // old embeddings - object TEXT_SIMILARITY_DAVINCI_001 : ModelType("text-similarity-davinci-001", R50K_BASE, 2046) - object TEXT_SIMILARITY_CURIE_001 : ModelType("text-similarity-curie-001", R50K_BASE, 2046) - object TEXT_SIMILARITY_BABBAGE_001 : ModelType("text-similarity-babbage-001", R50K_BASE, 2046) - object TEXT_SIMILARITY_ADA_001 : ModelType("text-similarity-ada-001", R50K_BASE, 2046) - object TEXT_SEARCH_DAVINCI_DOC_001 : ModelType("text-search-davinci-doc-001", R50K_BASE, 2046) - object TEXT_SEARCH_CURIE_DOC_001 : ModelType("text-search-curie-doc-001", R50K_BASE, 2046) - object TEXT_SEARCH_BABBAGE_DOC_001 : ModelType("text-search-babbage-doc-001", R50K_BASE, 2046) - object TEXT_SEARCH_ADA_DOC_001 : ModelType("text-search-ada-doc-001", R50K_BASE, 2046) - object CODE_SEARCH_BABBAGE_CODE_001 : ModelType("code-search-babbage-code-001", R50K_BASE, 2046) - object CODE_SEARCH_ADA_CODE_001 : ModelType("code-search-ada-code-001", R50K_BASE, 2046) - - /** - * Currently as of September 2023, - * [ModelType] has only implementations for OpenAI. - * Porting this class to other providers will hopefully be addressed in another PR. - * Meanwhile, [ModelType.TODO] serves as a placeholder. - */ - class TODO(name: String) : ModelType(name = name, encodingType = EncodingType.CL100K_BASE, maxContextLength = 2048) - - inline val encoding: Encoding - inline get() = encodingType.encoding - -} +//package com.xebia.functional.tokenizer +// +//import com.xebia.functional.tokenizer.EncodingType.CL100K_BASE +//import com.xebia.functional.tokenizer.EncodingType.P50K_BASE +//import com.xebia.functional.tokenizer.EncodingType.R50K_BASE +//import kotlin.jvm.JvmStatic +// +///** +// * Formal description of a model and it's properties +// * without any capabilities. +// */ +//open class ModelType( +// /** +// * Returns the name of the model type as used by the OpenAI API. +// * +// * @return the name of the model type +// */ +// open val name: String, +// +// open val maxInputTokens: Int, +// open val maxOutputTokens: Int, +// +// open val tokensPerMessage: Int = 0, +// open val tokensPerName: Int = 0, +// open val tokenPadding: Int = 20, +//) { +// +// /** +// * Returns the maximum context length that is supported by this model type. Note that +// * the maximum context length consists of the amount of prompt tokens and the amount of +// * completion tokens (where applicable). +// * +// * @return the maximum context length for this model type +// */ +// val maxContextLength: Int get() = maxInputTokens + maxOutputTokens +// +// val encoding: Encoding get() = TODO() +// +// data class LocalModel(override val name: String, override val maxContextLength: Int) : +// ModelType(name, maxContextLength) +// +//} From dc426c3302e90973ce175f5adf811d8125733c23 Mon Sep 17 00:00:00 2001 From: ron Date: Wed, 4 Oct 2023 13:25:48 +0200 Subject: [PATCH 02/19] one step closer to make code compiling --- .../kotlin/com/xebia/functional/xef/llm/Chat.kt | 5 +++++ .../functional/xef/llm/ChatWithFunctions.kt | 5 +++++ .../com/xebia/functional/xef/llm/Completion.kt | 2 ++ .../kotlin/com/xebia/functional/xef/llm/LLM.kt | 13 +++++++------ .../xebia/functional/xef/llm/PromptCalculator.kt | 4 ++-- .../xef/textsplitters/TokenTextSplitter.kt | 1 - .../xef/conversation/ConversationSpec.kt | 1 - .../xebia/functional/xef/data/TestEmbeddings.kt | 6 +++--- .../functional/xef/data/TestFunctionsModel.kt | 6 +++--- .../com/xebia/functional/xef/data/TestModel.kt | 6 +++--- .../xef/store/CombinedVectorStoreSpec.kt | 1 - .../functional/xef/store/LocalVectorStoreSpec.kt | 1 - .../com/xebia/functional/gpt4all/GPT4All.kt | 2 +- .../xebia/functional/xef/gcp/models/GcpChat.kt | 2 +- .../functional/xef/gcp/models/GcpCompletion.kt | 1 - .../functional/xef/gcp/models/GcpEmbeddings.kt | 6 +++--- .../com/xebia/functional/xef/pdf/PDFLoader.kt | 1 - .../src/test/kotlin/xef/PGVectorStoreSpec.kt | 15 +++++++++------ .../xef/conversation/llm/openai/OpenAI.kt | 2 +- .../conversation/llm/openai/models/OpenAIChat.kt | 14 +++++++++----- .../llm/openai/models/OpenAICompletion.kt | 12 ++++++++---- .../llm/openai/models/OpenAIEmbeddings.kt | 10 ++++++---- .../llm/openai/models/OpenAIFunChat.kt | 16 ++++++++++------ .../llm/openai/models/OpenAIImages.kt | 11 +++++++---- .../llm/openai/models/OpenAIModel.kt | 4 ++++ 25 files changed, 89 insertions(+), 58 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 13d59c28d..3b282ef63 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -3,6 +3,7 @@ package com.xebia.functional.xef.llm import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistant @@ -10,6 +11,10 @@ import kotlinx.coroutines.flow.* interface Chat : LLM { + val contextLength: MaxContextLength + + override val maxContextLength get() = (contextLength as MaxContextLength.Combined).total + suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse suspend fun createChatCompletions(request: ChatCompletionRequest): Flow diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index f6f98e9b2..a0e5a39d7 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -6,6 +6,7 @@ import arrow.core.raise.catch import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation +import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.llm.models.chat.ChatCompletionChunk import com.xebia.functional.xef.llm.models.chat.ChatCompletionResponseWithFunctions import com.xebia.functional.xef.llm.models.functions.CFunction @@ -21,6 +22,10 @@ import kotlinx.serialization.json.* interface ChatWithFunctions : LLM { + val contextLength: MaxContextLength + + override val maxContextLength get() = (contextLength as MaxContextLength.Combined).total + suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest ): ChatCompletionResponseWithFunctions diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index 6ae90a3b5..36b596a85 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -1,8 +1,10 @@ package com.xebia.functional.xef.llm +import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult interface Completion : LLM { + val contextLength: MaxContextLength suspend fun createCompletion(request: CompletionRequest): CompletionResult } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 87454401f..6af516771 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -17,13 +17,14 @@ import com.xebia.functional.xef.llm.models.chat.Message * Copies this instance and uses [modelType] for [LLM.modelType]. Has to return the most specific * type of this instance! */ - fun copy(modelType: ModelType): LLM + fun copy(modelID: ModelID): LLM - fun tokensFromMessages( - messages: List - ): Int { // TODO: naive implementation with magic numbers - TODO() - } + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") + fun tokensFromMessages(messages: List): Int = TODO() // intermediary + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") + fun truncateText(text: String, maxTokens: Int): String = TODO() // intermediary + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") + val maxContextLength: Int get() = TODO() override fun close() = Unit } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt index 9e115828a..a34d0a52f 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt @@ -44,7 +44,7 @@ internal object PromptCalculator { if (ctxInfo.isNotEmpty()) { val ctx: String = ctxInfo.joinToString("\n") - val ctxTruncated: String = llm.modelType.encoding.truncateText(ctx, maxContextTokens) + val ctxTruncated: String = llm.truncateText(ctx, maxContextTokens) Prompt { +assistant(ctxTruncated) }.messages } else { @@ -97,7 +97,7 @@ internal object PromptCalculator { } private fun calculateRemainingTokensForContext(llm: LLM, prompt: Prompt): Int { - val maxContextLength: Int = llm.modelType.maxContextLength + val maxContextLength: Int = llm.maxContextLength val remainingTokens: Int = maxContextLength - prompt.configuration.minResponseTokens val messagesTokens = llm.tokensFromMessages(prompt.messages) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt index 065505363..be831d4f5 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt @@ -1,7 +1,6 @@ package com.xebia.functional.xef.textsplitters import com.xebia.functional.tokenizer.Encoding -import com.xebia.functional.tokenizer.ModelType fun TokenTextSplitter(modelType: ModelType, chunkSize: Int, chunkOverlap: Int): TextSplitter = TokenTextSplitterImpl(modelType.encoding, chunkSize, chunkOverlap) diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index ffd818ecb..2f2994dbf 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -1,6 +1,5 @@ package com.xebia.functional.xef.conversation -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.data.* import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.chat.Role diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestEmbeddings.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestEmbeddings.kt index 2cc0d7989..48788d6c2 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestEmbeddings.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestEmbeddings.kt @@ -1,7 +1,7 @@ package com.xebia.functional.xef.data -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest import com.xebia.functional.xef.llm.models.embeddings.EmbeddingResult @@ -10,9 +10,9 @@ import com.xebia.functional.xef.llm.models.usage.Usage class TestEmbeddings : Embeddings { - override val modelType: ModelType = ModelType.TODO("test-embeddings") + override val modelID = ModelID("test-embeddings") - override fun copy(modelType: ModelType) = TestEmbeddings() + override fun copy(modelID: ModelID) = TestEmbeddings() override suspend fun embedDocuments( texts: List, diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt index eb04ff174..c90b741f3 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt @@ -1,8 +1,8 @@ package com.xebia.functional.xef.data -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.llm.ChatWithFunctions import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest import com.xebia.functional.xef.llm.models.embeddings.EmbeddingResult @@ -12,13 +12,13 @@ import com.xebia.functional.xef.llm.models.usage.Usage import kotlinx.coroutines.flow.Flow class TestFunctionsModel( - override val modelType: ModelType, + override val modelID: ModelID, val responses: Map = emptyMap(), ) : ChatWithFunctions, Embeddings, AutoCloseable { var requests: MutableList = mutableListOf() - override fun copy(modelType: ModelType) = TestFunctionsModel(modelType, responses) + override fun copy(modelID: ModelID) = TestFunctionsModel(modelID, responses) override fun tokensFromMessages(messages: List): Int { return messages.sumOf { it.content.length } diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt index 39d25569e..c947b14a0 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt @@ -1,8 +1,8 @@ package com.xebia.functional.xef.data -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest import com.xebia.functional.xef.llm.models.embeddings.EmbeddingResult @@ -10,13 +10,13 @@ import com.xebia.functional.xef.llm.models.usage.Usage import kotlinx.coroutines.flow.Flow class TestModel( - override val modelType: ModelType, + override val modelID: ModelID, val responses: Map = emptyMap(), ) : Chat, Embeddings, AutoCloseable { var requests: MutableList = mutableListOf() - override fun copy(modelType: ModelType) = TestModel(modelType, responses) + override fun copy(modelType: ModelID) = TestModel(modelType, responses) override suspend fun createChatCompletion( request: ChatCompletionRequest diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt index 9c86e93c2..63a276331 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt @@ -1,6 +1,5 @@ package com.xebia.functional.xef.store -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.data.TestEmbeddings import com.xebia.functional.xef.data.TestModel import io.kotest.core.spec.style.StringSpec diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt index 13ae49de5..fc4069cf6 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt @@ -1,6 +1,5 @@ package com.xebia.functional.xef.store -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.data.TestEmbeddings import com.xebia.functional.xef.data.TestModel import io.kotest.core.spec.style.StringSpec diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt index 710167ae9..644abb14d 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt @@ -67,7 +67,7 @@ interface GPT4All : AutoCloseable, Chat, Completion { override val modelID = ModelID(path.name) val llModel = LLModel(path) - override fun copy(modelType: ModelID) = + override fun copy(modelID: ModelID) = GPT4All(url, path) override suspend fun createCompletion(request: CompletionRequest): CompletionResult = diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt index c3e454c1f..8ac4a1bc2 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt @@ -20,7 +20,7 @@ class GcpChat( private val client = provider.defaultClient - override fun copy(modelType: ModelType) = GcpChat(provider, modelType) + override fun copy(modelID: ModelID) = GcpChat(provider, modelID) override suspend fun createChatCompletion( request: ChatCompletionRequest diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt index db9b3e7f8..b5c222a41 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt @@ -1,6 +1,5 @@ package com.xebia.functional.xef.gcp.models -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.gcp.GCP import com.xebia.functional.xef.llm.Completion import com.xebia.functional.xef.llm.models.text.CompletionChoice diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpEmbeddings.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpEmbeddings.kt index d46015694..62337318a 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpEmbeddings.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpEmbeddings.kt @@ -1,9 +1,9 @@ package com.xebia.functional.xef.gcp.models -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.gcp.GCP import com.xebia.functional.xef.gcp.GcpClient import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest import com.xebia.functional.xef.llm.models.embeddings.EmbeddingResult @@ -11,12 +11,12 @@ import com.xebia.functional.xef.llm.models.usage.Usage class GcpEmbeddings( private val provider: GCP, // TODO: use context receiver - override val modelType: ModelType, + override val modelID: ModelID, ) : Embeddings { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = GcpEmbeddings(provider, modelType) + override fun copy(modelID: ModelID) = GcpEmbeddings(provider, modelID) override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { fun requestToEmbedding(it: GcpClient.EmbeddingPredictions): Embedding = diff --git a/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt b/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt index 755053c5c..683af5b7c 100644 --- a/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt +++ b/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt @@ -2,7 +2,6 @@ @file:JvmMultifileClass package com.xebia.functional.xef.pdf -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.loaders.BaseLoader import com.xebia.functional.xef.textsplitters.TextSplitter import com.xebia.functional.xef.textsplitters.TokenTextSplitter diff --git a/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt b/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt index 8e95d55d1..387837cdc 100644 --- a/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt +++ b/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt @@ -1,9 +1,9 @@ package xef -import com.xebia.functional.tokenizer.ModelType import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Embeddings import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest @@ -109,9 +109,12 @@ class PGVectorStoreSpec : } }) -class TestLLM(override val modelType: ModelType = ModelType.ADA) : Chat, AutoCloseable { - override fun copy(modelType: ModelType) = - TestLLM(modelType) +class TestLLM : Chat, AutoCloseable { + + override val modelID = ModelID("test-llm") + + override fun copy(modelID: ModelID) = + TestLLM() override fun tokensFromMessages(messages: List): Int = messages.map { calculateTokens(it) }.sum() @@ -148,7 +151,7 @@ private fun Embeddings.Companion.mock( } ): Embeddings = object : Embeddings { - override fun copy(modelType: ModelType): LLM { + override fun copy(modelID: ModelID): LLM { throw NotImplementedError() } override suspend fun embedDocuments( @@ -164,5 +167,5 @@ private fun Embeddings.Companion.mock( createEmbeddings(request) - override val modelType: ModelType = ModelType.TODO("test-embeddings") + override val modelID = ModelID("test-embeddings") } diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 936b86e37..c48bd4592 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -208,7 +208,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu baseModel: T ): T { // TODO: impl of abstract provider function if (findModel(modelId) == null) error("model not found") - return baseModel.copy(ModelType.FineTunedModel(modelId, baseModel = baseModel.modelType)) as? T + return baseModel.copy(ModelID(modelId)) as? T ?: error("${baseModel::class} does not follow contract to return the most specific type") } diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt index 47d559d5f..ad5e2bdff 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt @@ -4,23 +4,27 @@ import com.aallam.openai.api.chat.ChatChoice import com.aallam.openai.api.chat.ChatMessage import com.aallam.openai.api.chat.chatCompletionRequest import com.aallam.openai.api.model.ModelId -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.conversation.llm.openai.toOpenAI import com.xebia.functional.xef.llm.Chat +import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map class OpenAIChat( private val provider: OpenAI, // TODO: use context receiver - override val modelType: ModelType, -) : Chat { + override val modelID: ModelID, + override val contextLength: MaxContextLength, + override val encodingType: EncodingType, +) : Chat, OpenAIModel { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = OpenAIChat(provider, modelType) + override fun copy(modelID: ModelID) = OpenAIChat(provider, modelID, contextLength, encodingType) override suspend fun createChatCompletion( request: ChatCompletionRequest @@ -43,7 +47,7 @@ class OpenAIChat( } private fun ChatCompletionRequest.toOpenAI() = chatCompletionRequest { - model = ModelId(this@OpenAIChat.modelType.name) + model = ModelId(this@OpenAIChat.modelID.value) messages = this@toOpenAI.messages.map { ChatMessage( diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt index e09a179b5..a431255a1 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt @@ -4,22 +4,26 @@ import com.aallam.openai.api.LegacyOpenAI import com.aallam.openai.api.completion.Choice import com.aallam.openai.api.completion.completionRequest import com.aallam.openai.api.model.ModelId -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.llm.Completion +import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult class OpenAICompletion( private val provider: OpenAI, // TODO: use context receiver - override val modelType: ModelType, -) : Completion { + override val modelID: ModelID, + override val contextLength: MaxContextLength, + override val encodingType: EncodingType, +) : Completion, OpenAIModel { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = OpenAICompletion(provider, modelType) + override fun copy(modelID: ModelID) = OpenAICompletion(provider, modelID, contextLength, encodingType) @OptIn(LegacyOpenAI::class) override suspend fun createCompletion(request: CompletionRequest): CompletionResult { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIEmbeddings.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIEmbeddings.kt index 7a144b3ef..cb2ae7f7a 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIEmbeddings.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIEmbeddings.kt @@ -3,22 +3,24 @@ package com.xebia.functional.xef.conversation.llm.openai.models import com.aallam.openai.api.embedding.Embedding as OpenAIEmbedding import com.aallam.openai.api.embedding.embeddingRequest import com.aallam.openai.api.model.ModelId -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest import com.xebia.functional.xef.llm.models.embeddings.EmbeddingResult class OpenAIEmbeddings( private val provider: OpenAI, // TODO: use context receiver - override val modelType: ModelType, -) : Embeddings { + override val modelID: ModelID, + override val encodingType: EncodingType, +) : Embeddings, OpenAIModel { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = OpenAIEmbeddings(provider, modelType) + override fun copy(modelID: ModelID) = OpenAIEmbeddings(provider, modelID, encodingType) override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { val clientRequest = embeddingRequest { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt index d7e0bd437..9877782ec 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt @@ -2,11 +2,13 @@ package com.xebia.functional.xef.conversation.llm.openai.models import com.aallam.openai.api.chat.* import com.aallam.openai.api.model.ModelId -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.conversation.llm.openai.toOpenAI import com.xebia.functional.xef.llm.ChatWithFunctions +import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.chat.ChatCompletionChunk import com.xebia.functional.xef.llm.models.functions.CFunction @@ -18,18 +20,20 @@ import kotlinx.serialization.json.Json class OpenAIFunChat( private val provider: OpenAI, // TODO: use context receiver - override val modelType: ModelType, -) : ChatWithFunctions { + override val modelID: ModelID, + override val contextLength: MaxContextLength, + override val encodingType: EncodingType, +) : ChatWithFunctions, OpenAIModel { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = OpenAIFunChat(provider, modelType) + override fun copy(modelID: ModelID) = OpenAIFunChat(provider, modelID, contextLength, encodingType) override suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest ): ChatCompletionResponseWithFunctions { val openAIRequest = chatCompletionRequest { - model = ModelId(modelType.name) + model = ModelId(modelID.value) messages = request.messages.map { it.toOpenAI() } functions = request.functions.map { it.toOpenAI() } temperature = request.temperature @@ -77,7 +81,7 @@ class OpenAIFunChat( request: FunChatCompletionRequest ): Flow { val clientRequest = chatCompletionRequest { - model = ModelId(modelType.name) + model = ModelId(modelID.value) messages = request.messages.map { it.toOpenAI() } temperature = request.temperature topP = request.topP diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt index b98a90fbc..d189cca17 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt @@ -4,9 +4,11 @@ import com.aallam.openai.api.BetaOpenAI import com.aallam.openai.api.image.ImageCreation import com.aallam.openai.api.image.ImageSize import com.aallam.openai.api.image.imageCreation -import com.xebia.functional.tokenizer.ModelType +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.llm.Images +import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.images.ImageGenerationUrl import com.xebia.functional.xef.llm.models.images.ImagesGenerationRequest @@ -14,12 +16,13 @@ import com.xebia.functional.xef.llm.models.images.ImagesGenerationResponse class OpenAIImages( private val provider: OpenAI, // TODO: use context receiver - override val modelType: ModelType, -) : Images { + override val modelID: ModelID, + override val encodingType: EncodingType, +) : Images, OpenAIModel { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = OpenAIImages(provider, modelType) + override fun copy(modelID: ModelID) = OpenAIImages(provider, modelID, encodingType) @OptIn(BetaOpenAI::class) override suspend fun createImages(request: ImagesGenerationRequest): ImagesGenerationResponse { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt index 693fdd334..8f4372314 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt @@ -2,6 +2,7 @@ package com.xebia.functional.xef.conversation.llm.openai.models import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.tokenizer.EncodingType +import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.llm.LLM import com.xebia.functional.xef.llm.models.chat.Message @@ -24,4 +25,7 @@ interface OpenAIModel : LLM { tokensPerName = 5 ) + 10 } + + override fun truncateText(text: String, maxTokens: Int): String = + encoding.truncateText(text, maxTokens) } From 004f980799cad4845d48e942caaa64afa3a831fb Mon Sep 17 00:00:00 2001 From: ron Date: Wed, 4 Oct 2023 15:00:24 +0200 Subject: [PATCH 03/19] one step closer to make code compiling --- .../xef/textsplitters/TokenTextSplitter.kt | 5 ++-- .../com/xebia/functional/gpt4all/GPT4All.kt | 3 ++ .../functional/xef/gcp/models/GcpChat.kt | 4 ++- .../xef/gcp/models/GcpCompletion.kt | 13 +++++---- .../com/xebia/functional/xef/pdf/PDFLoader.kt | 5 ++-- .../xef/conversation/llm/openai/OpenAI.kt | 18 ++++++------ .../xef/reasoning/text/summarize/Summarize.kt | 28 +++++++++++-------- 7 files changed, 46 insertions(+), 30 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt index be831d4f5..ef09a8d40 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/textsplitters/TokenTextSplitter.kt @@ -1,9 +1,10 @@ package com.xebia.functional.xef.textsplitters import com.xebia.functional.tokenizer.Encoding +import com.xebia.functional.tokenizer.EncodingType -fun TokenTextSplitter(modelType: ModelType, chunkSize: Int, chunkOverlap: Int): TextSplitter = - TokenTextSplitterImpl(modelType.encoding, chunkSize, chunkOverlap) +fun TokenTextSplitter(encodingType: EncodingType, chunkSize: Int, chunkOverlap: Int): TextSplitter = + TokenTextSplitterImpl(encodingType.encoding, chunkSize, chunkOverlap) private class TokenTextSplitterImpl( private val tokenizer: Encoding, diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt index 644abb14d..2151deae5 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt @@ -8,6 +8,7 @@ import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.conversation.PlatformConversation import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Completion +import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.text.CompletionChoice @@ -64,6 +65,8 @@ interface GPT4All : AutoCloseable, Chat, Completion { } } + override val contextLength = MaxContextLength.Combined(2048) //TODO: no hardcode + override val modelID = ModelID(path.name) val llModel = LLModel(path) diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt index 8ac4a1bc2..10c58c242 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt @@ -3,6 +3,7 @@ package com.xebia.functional.xef.gcp.models import com.xebia.functional.xef.gcp.GCP import com.xebia.functional.xef.gcp.GcpClient import com.xebia.functional.xef.llm.Chat +import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.usage.Usage @@ -16,11 +17,12 @@ import kotlinx.uuid.generateUUID class GcpChat( private val provider: GCP, // TODO: use context receiver override val modelID: ModelID, + override val contextLength: MaxContextLength ) : Chat { private val client = provider.defaultClient - override fun copy(modelID: ModelID) = GcpChat(provider, modelID) + override fun copy(modelID: ModelID) = GcpChat(provider, modelID, contextLength) override suspend fun createChatCompletion( request: ChatCompletionRequest diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt index b5c222a41..01215e3b9 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt @@ -2,6 +2,8 @@ package com.xebia.functional.xef.gcp.models import com.xebia.functional.xef.gcp.GCP import com.xebia.functional.xef.llm.Completion +import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult @@ -12,17 +14,18 @@ import kotlinx.uuid.generateUUID class GcpCompletion( private val provider: GCP, // TODO: use context receiver - override val modelType: ModelType, + override val modelID: ModelID, + override val contextLength: MaxContextLength, ) : Completion { private val client = provider.defaultClient - override fun copy(modelType: ModelType) = GcpCompletion(provider, modelType) + override fun copy(modelID: ModelID) = GcpCompletion(provider, modelID, contextLength) override suspend fun createCompletion(request: CompletionRequest): CompletionResult { val response: String = client.promptMessage( - modelType.name, + modelID.value, request.prompt, temperature = request.temperature, maxOutputTokens = request.maxTokens, @@ -30,9 +33,9 @@ class GcpCompletion( ) return CompletionResult( UUID.generateUUID().toString(), - modelType.name, + modelID.value, getTimeMillis(), - modelType.name, + modelID.value, listOf(CompletionChoice(response, 0, null, null)), Usage.ZERO, // TODO: token usage - no information about usage provided by GCP codechat model ) diff --git a/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt b/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt index 683af5b7c..5c63b971b 100644 --- a/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt +++ b/integrations/pdf/src/main/kotlin/com/xebia/functional/xef/pdf/PDFLoader.kt @@ -2,6 +2,7 @@ @file:JvmMultifileClass package com.xebia.functional.xef.pdf +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.loaders.BaseLoader import com.xebia.functional.xef.textsplitters.TextSplitter import com.xebia.functional.xef.textsplitters.TokenTextSplitter @@ -17,7 +18,7 @@ import java.io.File suspend fun pdf( url: String, - splitter: TextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50) + splitter: TextSplitter = TokenTextSplitter(encodingType = EncodingType.CL100K_BASE, chunkSize = 100, chunkOverlap = 50) ): List = HttpClient().use { val response = it.get(url) @@ -30,7 +31,7 @@ suspend fun pdf( suspend fun pdf( file: File, - splitter: TextSplitter = TokenTextSplitter(modelType = ModelType.GPT_3_5_TURBO, chunkSize = 100, chunkOverlap = 50) + splitter: TextSplitter = TokenTextSplitter(encodingType = EncodingType.CL100K_BASE, chunkSize = 100, chunkOverlap = 50) ): List { val loader = PDFLoader(file) return loader.loadAndSplit(splitter) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index c48bd4592..5214120b9 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -82,7 +82,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu } val GPT_4_0314 by lazy { - OpenAIFunChat(this, ModelID("gpt-4-0314"), EncodingType.CL100K_BASE) + OpenAIFunChat(this, ModelID("gpt-4-0314"), MaxContextLength.Combined(4097), EncodingType.CL100K_BASE) .autoCloseBind() // legacy } @@ -117,7 +117,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu } val GPT_3_5_TURBO_FUNCTIONS by lazy { - OpenAIFunChat(this, ModelID("gpt-3.5-turbo-0613"), EncodingType.CL100K_BASE) + OpenAIFunChat(this, ModelID("gpt-3.5-turbo-0613"), MaxContextLength.Combined(4097), EncodingType.CL100K_BASE) .autoCloseBind() } @@ -132,27 +132,27 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu } val TEXT_DAVINCI_003 by lazy { - OpenAICompletion(this, ModelID("text-davinci-003"), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-davinci-003"), MaxContextLength.Combined(4097), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_DAVINCI_002 by lazy { - OpenAICompletion(this, ModelID("text-davinci-002"), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-davinci-002"), MaxContextLength.Combined(4097), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_CURIE_001 by lazy { - OpenAICompletion(this, ModelID("text-similarity-curie-001"), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-similarity-curie-001"), MaxContextLength.Combined(2049), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_BABBAGE_001 by lazy { - OpenAICompletion(this, ModelID("text-babbage-001"), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-babbage-001"), MaxContextLength.Combined(2049), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_ADA_001 by lazy { - OpenAICompletion(this, ModelID("text-ada-001"), EncodingType.P50K_BASE).autoCloseBind() + OpenAICompletion(this, ModelID("text-ada-001"), MaxContextLength.Combined(2049), EncodingType.P50K_BASE).autoCloseBind() } val TEXT_EMBEDDING_ADA_002 by lazy { @@ -190,7 +190,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu DALLE_2, ) - suspend fun findModel(modelId: String): Any? { // TODO: impl of abstract provider function + suspend fun findModel(modelId: String): ModelID? { // TODO: impl of abstract provider function val model = try { defaultClient.model(ModelId(modelId)) @@ -200,7 +200,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu else -> throw e } } - return ModelType.TODO(model.id.id) + return ModelID(model.id.id) } suspend fun spawnModel( diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt index 8cb38985b..b22e283d1 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt @@ -1,9 +1,11 @@ package com.xebia.functional.xef.reasoning.text.summarize import arrow.fx.coroutines.parMap +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.llm.Chat +import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistantSteps import com.xebia.functional.xef.prompt.templates.system @@ -38,9 +40,11 @@ class Summarize( summarizeLargeText(text = input, summaryLength = summaryLength) private suspend fun summarizeChunk(chunk: String, summaryLength: SummaryLength): String { - val maxContextLength: Int = - model.modelType.maxContextLength - 1000 // magic padding for functions and memory - val promptTokens: Int = model.modelType.encoding.countTokens(chunk) + val maxContextLength = when(val contextLength = model.contextLength) { + is MaxContextLength.Combined -> contextLength.total + is MaxContextLength.FixIO -> contextLength.input + } - 1000 // magic padding for functions and memory + val promptTokens: Int = encodingType.encoding.countTokens(chunk) logger.info { "📝 Summarizing chunk with prompt tokens $promptTokens for length $summaryLength" } @@ -54,7 +58,7 @@ class Summarize( """| |Given the following text: |```text - |${model.modelType.encoding.truncateText(chunk, remainingTokens)} + |${encodingType.encoding.truncateText(chunk, remainingTokens)} |``` """ .trimMargin() @@ -68,15 +72,17 @@ class Summarize( } return model.promptMessage(messages, scope).also { - val tokens: Int = model.modelType.encoding.countTokens(it) + val tokens: Int = encodingType.encoding.countTokens(it) logger.info { "📝 Summarized chunk in tokens: $tokens" } } } - private fun chunkText(text: String): List { - val maxTokens = - model.modelType.maxContextLength - 2000 // magic padding for functions and memory - val firstPart = model.modelType.encoding.truncateText(text, maxTokens) + private suspend fun chunkText(text: String): List { + val maxTokens = when(val contextLength = model.contextLength) { + is MaxContextLength.Combined -> contextLength.total + is MaxContextLength.FixIO -> contextLength.input + } - 2000 // magic padding for functions and memory + val firstPart = model.truncateText(text, maxTokens) val remainingText = text.removePrefix(firstPart) return if (remainingText.isNotEmpty()) { @@ -87,7 +93,7 @@ class Summarize( } tailrec suspend fun summarizeLargeText(text: String, summaryLength: SummaryLength): String { - val tokens = model.modelType.encoding.countTokens(text) + val tokens = encodingType.encoding.countTokens(text) logger.info { "📚 Summarizing large text of tokens ${tokens} to approximately $summaryLength tokens" } @@ -110,7 +116,7 @@ class Summarize( // Join the chunk summaries into one text val joinedSummaries = chunkSummaries.joinToString(" ") - val joinedSummariesTokens = model.modelType.encoding.countTokens(joinedSummaries) + val joinedSummariesTokens = encodingType.encoding.countTokens(joinedSummaries) // Resummarize the joined summaries if it is longer than summaryLength return if ( From eea22e819afc1dfb6b6e6f087757375cf3f40f5e Mon Sep 17 00:00:00 2001 From: ron Date: Wed, 4 Oct 2023 15:04:52 +0200 Subject: [PATCH 04/19] delete ModelType.kt --- .../xebia/functional/tokenizer/ModelType.kt | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt diff --git a/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt b/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt deleted file mode 100644 index 726059d09..000000000 --- a/tokenizer/src/commonMain/kotlin/com/xebia/functional/tokenizer/ModelType.kt +++ /dev/null @@ -1,42 +0,0 @@ -//package com.xebia.functional.tokenizer -// -//import com.xebia.functional.tokenizer.EncodingType.CL100K_BASE -//import com.xebia.functional.tokenizer.EncodingType.P50K_BASE -//import com.xebia.functional.tokenizer.EncodingType.R50K_BASE -//import kotlin.jvm.JvmStatic -// -///** -// * Formal description of a model and it's properties -// * without any capabilities. -// */ -//open class ModelType( -// /** -// * Returns the name of the model type as used by the OpenAI API. -// * -// * @return the name of the model type -// */ -// open val name: String, -// -// open val maxInputTokens: Int, -// open val maxOutputTokens: Int, -// -// open val tokensPerMessage: Int = 0, -// open val tokensPerName: Int = 0, -// open val tokenPadding: Int = 20, -//) { -// -// /** -// * Returns the maximum context length that is supported by this model type. Note that -// * the maximum context length consists of the amount of prompt tokens and the amount of -// * completion tokens (where applicable). -// * -// * @return the maximum context length for this model type -// */ -// val maxContextLength: Int get() = maxInputTokens + maxOutputTokens -// -// val encoding: Encoding get() = TODO() -// -// data class LocalModel(override val name: String, override val maxContextLength: Int) : -// ModelType(name, maxContextLength) -// -//} From 34411974281da2224d88d9510ed68fda61b79a45 Mon Sep 17 00:00:00 2001 From: ron Date: Wed, 4 Oct 2023 15:15:36 +0200 Subject: [PATCH 05/19] rename MaxContextLength.kt --- .../com/xebia/functional/xef/llm/Chat.kt | 6 ++--- .../functional/xef/llm/ChatWithFunctions.kt | 6 ++--- .../xebia/functional/xef/llm/Completion.kt | 4 +-- .../xef/llm/models/MaxContextLength.kt | 9 ------- .../xef/llm/models/MaxIoContextLength.kt | 15 +++++++++++ .../com/xebia/functional/gpt4all/GPT4All.kt | 5 ++-- .../functional/xef/gcp/models/GcpChat.kt | 6 ++--- .../xef/gcp/models/GcpCompletion.kt | 4 +-- .../xef/conversation/llm/openai/OpenAI.kt | 26 +++++++++---------- .../llm/openai/models/OpenAIChat.kt | 4 +-- .../llm/openai/models/OpenAICompletion.kt | 4 +-- .../llm/openai/models/OpenAIFunChat.kt | 4 +-- .../llm/openai/models/OpenAIImages.kt | 1 - .../xef/reasoning/text/summarize/Summarize.kt | 11 ++++---- 14 files changed, 53 insertions(+), 52 deletions(-) delete mode 100644 core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt create mode 100644 core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 3b282ef63..9db613a20 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -3,7 +3,7 @@ package com.xebia.functional.xef.llm import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistant @@ -11,9 +11,9 @@ import kotlinx.coroutines.flow.* interface Chat : LLM { - val contextLength: MaxContextLength + val contextLength: MaxIoContextLength - override val maxContextLength get() = (contextLength as MaxContextLength.Combined).total + override val maxContextLength get() = (contextLength as MaxIoContextLength.Combined).total suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index a0e5a39d7..6b27b4672 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -6,7 +6,7 @@ import arrow.core.raise.catch import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.chat.ChatCompletionChunk import com.xebia.functional.xef.llm.models.chat.ChatCompletionResponseWithFunctions import com.xebia.functional.xef.llm.models.functions.CFunction @@ -22,9 +22,9 @@ import kotlinx.serialization.json.* interface ChatWithFunctions : LLM { - val contextLength: MaxContextLength + val contextLength: MaxIoContextLength - override val maxContextLength get() = (contextLength as MaxContextLength.Combined).total + override val maxContextLength get() = (contextLength as MaxIoContextLength.Combined).total suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index 36b596a85..a0fb9d7a2 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -1,10 +1,10 @@ package com.xebia.functional.xef.llm -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult interface Completion : LLM { - val contextLength: MaxContextLength + val contextLength: MaxIoContextLength suspend fun createCompletion(request: CompletionRequest): CompletionResult } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt deleted file mode 100644 index bb32e416f..000000000 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxContextLength.kt +++ /dev/null @@ -1,9 +0,0 @@ -package com.xebia.functional.xef.llm.models - -sealed interface MaxContextLength { - /** one total length of input and output combined */ - data class Combined(val total: Int) : MaxContextLength - - /** two separate max lengths for input and output respectively */ - data class FixIO(val input: Int, val output: Int) : MaxContextLength -} diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt new file mode 100644 index 000000000..4bcdc0e4d --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt @@ -0,0 +1,15 @@ +package com.xebia.functional.xef.llm.models + +/** + * Describing the maximum context length a model + * with text input and output might have. + * + * Some models from VertexAI (in 2023/10) have both types of max context length. + */ +sealed interface MaxIoContextLength { + /** one total length of input and output combined */ + data class Combined(val total: Int) : MaxIoContextLength + + /** two separate max lengths for input and output respectively */ + data class Fix(val input: Int, val output: Int) : MaxIoContextLength +} diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt index 2151deae5..f6d08bf8f 100644 --- a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt +++ b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt @@ -3,12 +3,11 @@ package com.xebia.functional.gpt4all import ai.djl.training.util.DownloadUtils import ai.djl.training.util.ProgressBar import com.hexadevlabs.gpt4all.LLModel -import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.conversation.PlatformConversation import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Completion -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.text.CompletionChoice @@ -65,7 +64,7 @@ interface GPT4All : AutoCloseable, Chat, Completion { } } - override val contextLength = MaxContextLength.Combined(2048) //TODO: no hardcode + override val contextLength = MaxIoContextLength.Combined(2048) //TODO: no hardcode override val modelID = ModelID(path.name) val llModel = LLModel(path) diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt index 10c58c242..67ff5b12d 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt @@ -1,13 +1,11 @@ package com.xebia.functional.xef.gcp.models import com.xebia.functional.xef.gcp.GCP -import com.xebia.functional.xef.gcp.GcpClient import com.xebia.functional.xef.llm.Chat -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.usage.Usage -import io.ktor.client.* import io.ktor.util.date.* import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.flow @@ -17,7 +15,7 @@ import kotlinx.uuid.generateUUID class GcpChat( private val provider: GCP, // TODO: use context receiver override val modelID: ModelID, - override val contextLength: MaxContextLength + override val contextLength: MaxIoContextLength ) : Chat { private val client = provider.defaultClient diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt index 01215e3b9..a9c574787 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt @@ -2,7 +2,7 @@ package com.xebia.functional.xef.gcp.models import com.xebia.functional.xef.gcp.GCP import com.xebia.functional.xef.llm.Completion -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest @@ -15,7 +15,7 @@ import kotlinx.uuid.generateUUID class GcpCompletion( private val provider: GCP, // TODO: use context receiver override val modelID: ModelID, - override val contextLength: MaxContextLength, + override val contextLength: MaxIoContextLength, ) : Completion { private val client = provider.defaultClient diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 5214120b9..45949b5b1 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -16,7 +16,7 @@ import com.xebia.functional.xef.conversation.autoClose import com.xebia.functional.xef.conversation.llm.openai.models.* import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.llm.LLM -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore @@ -75,14 +75,14 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu OpenAIChat( this, ModelID("gpt-4"), - MaxContextLength.Combined(8192), + MaxIoContextLength.Combined(8192), EncodingType.CL100K_BASE ) .autoCloseBind() } val GPT_4_0314 by lazy { - OpenAIFunChat(this, ModelID("gpt-4-0314"), MaxContextLength.Combined(4097), EncodingType.CL100K_BASE) + OpenAIFunChat(this, ModelID("gpt-4-0314"), MaxIoContextLength.Combined(4097), EncodingType.CL100K_BASE) .autoCloseBind() // legacy } @@ -90,7 +90,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu OpenAIChat( this, ModelID("gpt-4-32k"), - MaxContextLength.Combined(32768), + MaxIoContextLength.Combined(32768), EncodingType.CL100K_BASE ) .autoCloseBind() @@ -100,7 +100,7 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu OpenAIChat( this, ModelID("gpt-3.5-turbo"), - MaxContextLength.Combined(4097), + MaxIoContextLength.Combined(4097), EncodingType.CL100K_BASE ) .autoCloseBind() @@ -110,14 +110,14 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu OpenAIChat( this, ModelID("gpt-3.5-turbo-16k"), - MaxContextLength.Combined(4097 * 4), + MaxIoContextLength.Combined(4097 * 4), EncodingType.CL100K_BASE ) .autoCloseBind() } val GPT_3_5_TURBO_FUNCTIONS by lazy { - OpenAIFunChat(this, ModelID("gpt-3.5-turbo-0613"), MaxContextLength.Combined(4097), EncodingType.CL100K_BASE) + OpenAIFunChat(this, ModelID("gpt-3.5-turbo-0613"), MaxIoContextLength.Combined(4097), EncodingType.CL100K_BASE) .autoCloseBind() } @@ -125,34 +125,34 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu OpenAIChat( this, ModelID("gpt-3.5-turbo-0301"), - MaxContextLength.Combined(4097), + MaxIoContextLength.Combined(4097), EncodingType.CL100K_BASE ) .autoCloseBind() // legacy } val TEXT_DAVINCI_003 by lazy { - OpenAICompletion(this, ModelID("text-davinci-003"), MaxContextLength.Combined(4097), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-davinci-003"), MaxIoContextLength.Combined(4097), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_DAVINCI_002 by lazy { - OpenAICompletion(this, ModelID("text-davinci-002"), MaxContextLength.Combined(4097), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-davinci-002"), MaxIoContextLength.Combined(4097), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_CURIE_001 by lazy { - OpenAICompletion(this, ModelID("text-similarity-curie-001"), MaxContextLength.Combined(2049), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-similarity-curie-001"), MaxIoContextLength.Combined(2049), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_BABBAGE_001 by lazy { - OpenAICompletion(this, ModelID("text-babbage-001"), MaxContextLength.Combined(2049), EncodingType.P50K_BASE) + OpenAICompletion(this, ModelID("text-babbage-001"), MaxIoContextLength.Combined(2049), EncodingType.P50K_BASE) .autoCloseBind() } val TEXT_ADA_001 by lazy { - OpenAICompletion(this, ModelID("text-ada-001"), MaxContextLength.Combined(2049), EncodingType.P50K_BASE).autoCloseBind() + OpenAICompletion(this, ModelID("text-ada-001"), MaxIoContextLength.Combined(2049), EncodingType.P50K_BASE).autoCloseBind() } val TEXT_EMBEDDING_ADA_002 by lazy { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt index ad5e2bdff..bee199434 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt @@ -9,7 +9,7 @@ import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.conversation.llm.openai.toOpenAI import com.xebia.functional.xef.llm.Chat -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import kotlinx.coroutines.flow.Flow @@ -18,7 +18,7 @@ import kotlinx.coroutines.flow.map class OpenAIChat( private val provider: OpenAI, // TODO: use context receiver override val modelID: ModelID, - override val contextLength: MaxContextLength, + override val contextLength: MaxIoContextLength, override val encodingType: EncodingType, ) : Chat, OpenAIModel { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt index a431255a1..ed7498de8 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt @@ -8,7 +8,7 @@ import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.llm.Completion -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest @@ -17,7 +17,7 @@ import com.xebia.functional.xef.llm.models.text.CompletionResult class OpenAICompletion( private val provider: OpenAI, // TODO: use context receiver override val modelID: ModelID, - override val contextLength: MaxContextLength, + override val contextLength: MaxIoContextLength, override val encodingType: EncodingType, ) : Completion, OpenAIModel { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt index 9877782ec..9c6518639 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt @@ -7,7 +7,7 @@ import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.conversation.llm.openai.toOpenAI import com.xebia.functional.xef.llm.ChatWithFunctions -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.chat.ChatCompletionChunk @@ -21,7 +21,7 @@ import kotlinx.serialization.json.Json class OpenAIFunChat( private val provider: OpenAI, // TODO: use context receiver override val modelID: ModelID, - override val contextLength: MaxContextLength, + override val contextLength: MaxIoContextLength, override val encodingType: EncodingType, ) : ChatWithFunctions, OpenAIModel { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt index d189cca17..3e17afc33 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt @@ -7,7 +7,6 @@ import com.aallam.openai.api.image.imageCreation import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.llm.Images -import com.xebia.functional.xef.llm.models.MaxContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.images.ImageGenerationUrl diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt index b22e283d1..929811cf7 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt @@ -1,11 +1,10 @@ package com.xebia.functional.xef.reasoning.text.summarize import arrow.fx.coroutines.parMap -import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.llm.Chat -import com.xebia.functional.xef.llm.models.MaxContextLength +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistantSteps import com.xebia.functional.xef.prompt.templates.system @@ -41,8 +40,8 @@ class Summarize( private suspend fun summarizeChunk(chunk: String, summaryLength: SummaryLength): String { val maxContextLength = when(val contextLength = model.contextLength) { - is MaxContextLength.Combined -> contextLength.total - is MaxContextLength.FixIO -> contextLength.input + is MaxIoContextLength.Combined -> contextLength.total + is MaxIoContextLength.Fix -> contextLength.input } - 1000 // magic padding for functions and memory val promptTokens: Int = encodingType.encoding.countTokens(chunk) logger.info { @@ -79,8 +78,8 @@ class Summarize( private suspend fun chunkText(text: String): List { val maxTokens = when(val contextLength = model.contextLength) { - is MaxContextLength.Combined -> contextLength.total - is MaxContextLength.FixIO -> contextLength.input + is MaxIoContextLength.Combined -> contextLength.total + is MaxIoContextLength.Fix -> contextLength.input } - 2000 // magic padding for functions and memory val firstPart = model.truncateText(text, maxTokens) val remainingText = text.removePrefix(firstPart) From aea81bb07f3b36d78c1cac315e1e9af60201dfd9 Mon Sep 17 00:00:00 2001 From: ron Date: Wed, 4 Oct 2023 15:38:10 +0200 Subject: [PATCH 06/19] spotless --- .../com/xebia/functional/xef/llm/Chat.kt | 3 +- .../functional/xef/llm/ChatWithFunctions.kt | 3 +- .../xebia/functional/xef/llm/Completion.kt | 1 + .../com/xebia/functional/xef/llm/LLM.kt | 11 +- .../xef/llm/models/MaxIoContextLength.kt | 3 +- .../com/xebia/functional/xef/gcp/GCP.kt | 4 +- .../xef/conversation/llm/openai/OpenAI.kt | 103 +++++++++++++----- .../llm/openai/models/OpenAICompletion.kt | 3 +- .../llm/openai/models/OpenAIFunChat.kt | 3 +- .../llm/openai/models/OpenAIModel.kt | 11 +- .../xef/reasoning/text/summarize/Summarize.kt | 18 +-- 11 files changed, 106 insertions(+), 57 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 9db613a20..82187cba8 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -13,7 +13,8 @@ interface Chat : LLM { val contextLength: MaxIoContextLength - override val maxContextLength get() = (contextLength as MaxIoContextLength.Combined).total + override val maxContextLength + get() = (contextLength as MaxIoContextLength.Combined).total suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index 6b27b4672..825810f8b 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -24,7 +24,8 @@ interface ChatWithFunctions : LLM { val contextLength: MaxIoContextLength - override val maxContextLength get() = (contextLength as MaxIoContextLength.Combined).total + override val maxContextLength + get() = (contextLength as MaxIoContextLength.Combined).total suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index a0fb9d7a2..359a565e7 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -6,5 +6,6 @@ import com.xebia.functional.xef.llm.models.text.CompletionResult interface Completion : LLM { val contextLength: MaxIoContextLength + suspend fun createCompletion(request: CompletionRequest): CompletionResult } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 6af516771..bb98a5d21 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -1,11 +1,11 @@ package com.xebia.functional.xef.llm -import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.Message -// sealed modifier temporarily removed as OAI's implementation of tokensFromMessages has to extend and override LLM -/*sealed */interface LLM : AutoCloseable { +// sealed modifier temporarily removed as OAI's implementation of tokensFromMessages has to extend +// and override LLM +/*sealed */ interface LLM : AutoCloseable { val modelID: ModelID @@ -21,10 +21,13 @@ import com.xebia.functional.xef.llm.models.chat.Message @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") fun tokensFromMessages(messages: List): Int = TODO() // intermediary + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") fun truncateText(text: String, maxTokens: Int): String = TODO() // intermediary + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") - val maxContextLength: Int get() = TODO() + val maxContextLength: Int + get() = TODO() override fun close() = Unit } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt index 4bcdc0e4d..99673c167 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/models/MaxIoContextLength.kt @@ -1,8 +1,7 @@ package com.xebia.functional.xef.llm.models /** - * Describing the maximum context length a model - * with text input and output might have. + * Describing the maximum context length a model with text input and output might have. * * Some models from VertexAI (in 2023/10) have both types of max context length. */ diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt index df808d823..e95214f67 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt @@ -48,9 +48,7 @@ class GCP(projectId: String? = null, location: VertexAIRegion? = null, token: St val defaultClient = GcpClient(config) val CODECHAT by lazy { GcpChat(this, ModelID("codechat-bison@001")) } - val TEXT_EMBEDDING_GECKO by lazy { - GcpEmbeddings(this, ModelID("textembedding-gecko")) - } + val TEXT_EMBEDDING_GECKO by lazy { GcpEmbeddings(this, ModelID("textembedding-gecko")) } @JvmField val DEFAULT_CHAT = CODECHAT @JvmField val DEFAULT_EMBEDDING = TEXT_EMBEDDING_GECKO diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 45949b5b1..89fc47531 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -73,95 +73,140 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu val GPT_4 by lazy { OpenAIChat( - this, - ModelID("gpt-4"), - MaxIoContextLength.Combined(8192), - EncodingType.CL100K_BASE + provider = this, + modelID = ModelID("gpt-4"), + contextLength = MaxIoContextLength.Combined(8192), + encodingType = EncodingType.CL100K_BASE ) .autoCloseBind() } val GPT_4_0314 by lazy { - OpenAIFunChat(this, ModelID("gpt-4-0314"), MaxIoContextLength.Combined(4097), EncodingType.CL100K_BASE) + OpenAIFunChat( + provider = this, + modelID = ModelID("gpt-4-0314"), + contextLength = MaxIoContextLength.Combined(4097), + encodingType = EncodingType.CL100K_BASE + ) .autoCloseBind() // legacy } val GPT_4_32K by lazy { OpenAIChat( - this, - ModelID("gpt-4-32k"), - MaxIoContextLength.Combined(32768), - EncodingType.CL100K_BASE + provider = this, + modelID = ModelID("gpt-4-32k"), + contextLength = MaxIoContextLength.Combined(32768), + encodingType = EncodingType.CL100K_BASE ) .autoCloseBind() } val GPT_3_5_TURBO by lazy { OpenAIChat( - this, - ModelID("gpt-3.5-turbo"), - MaxIoContextLength.Combined(4097), - EncodingType.CL100K_BASE + provider = this, + modelID = ModelID("gpt-3.5-turbo"), + contextLength = MaxIoContextLength.Combined(4097), + encodingType = EncodingType.CL100K_BASE ) .autoCloseBind() } val GPT_3_5_TURBO_16K by lazy { OpenAIChat( - this, - ModelID("gpt-3.5-turbo-16k"), - MaxIoContextLength.Combined(4097 * 4), - EncodingType.CL100K_BASE + provider = this, + modelID = ModelID("gpt-3.5-turbo-16k"), + contextLength = MaxIoContextLength.Combined(4097 * 4), + encodingType = EncodingType.CL100K_BASE ) .autoCloseBind() } val GPT_3_5_TURBO_FUNCTIONS by lazy { - OpenAIFunChat(this, ModelID("gpt-3.5-turbo-0613"), MaxIoContextLength.Combined(4097), EncodingType.CL100K_BASE) + OpenAIFunChat( + provider = this, + modelID = ModelID("gpt-3.5-turbo-0613"), + contextLength = MaxIoContextLength.Combined(4097), + encodingType = EncodingType.CL100K_BASE + ) .autoCloseBind() } val GPT_3_5_TURBO_0301 by lazy { OpenAIChat( - this, - ModelID("gpt-3.5-turbo-0301"), - MaxIoContextLength.Combined(4097), - EncodingType.CL100K_BASE + provider = this, + modelID = ModelID("gpt-3.5-turbo-0301"), + contextLength = MaxIoContextLength.Combined(4097), + encodingType = EncodingType.CL100K_BASE ) .autoCloseBind() // legacy } val TEXT_DAVINCI_003 by lazy { - OpenAICompletion(this, ModelID("text-davinci-003"), MaxIoContextLength.Combined(4097), EncodingType.P50K_BASE) + OpenAICompletion( + provider = this, + modelID = ModelID("text-davinci-003"), + contextLength = MaxIoContextLength.Combined(4097), + encodingType = EncodingType.P50K_BASE + ) .autoCloseBind() } val TEXT_DAVINCI_002 by lazy { - OpenAICompletion(this, ModelID("text-davinci-002"), MaxIoContextLength.Combined(4097), EncodingType.P50K_BASE) + OpenAICompletion( + provider = this, + modelID = ModelID("text-davinci-002"), + contextLength = MaxIoContextLength.Combined(4097), + encodingType = EncodingType.P50K_BASE + ) .autoCloseBind() } val TEXT_CURIE_001 by lazy { - OpenAICompletion(this, ModelID("text-similarity-curie-001"), MaxIoContextLength.Combined(2049), EncodingType.P50K_BASE) + OpenAICompletion( + provider = this, + modelID = ModelID("text-similarity-curie-001"), + contextLength = MaxIoContextLength.Combined(2049), + encodingType = EncodingType.P50K_BASE + ) .autoCloseBind() } val TEXT_BABBAGE_001 by lazy { - OpenAICompletion(this, ModelID("text-babbage-001"), MaxIoContextLength.Combined(2049), EncodingType.P50K_BASE) + OpenAICompletion( + provider = this, + modelID = ModelID("text-babbage-001"), + contextLength = MaxIoContextLength.Combined(2049), + encodingType = EncodingType.P50K_BASE + ) .autoCloseBind() } val TEXT_ADA_001 by lazy { - OpenAICompletion(this, ModelID("text-ada-001"), MaxIoContextLength.Combined(2049), EncodingType.P50K_BASE).autoCloseBind() + OpenAICompletion( + provider = this, + modelID = ModelID("text-ada-001"), + contextLength = MaxIoContextLength.Combined(2049), + encodingType = EncodingType.P50K_BASE + ) + .autoCloseBind() } val TEXT_EMBEDDING_ADA_002 by lazy { - OpenAIEmbeddings(this, ModelID("text-embedding-ada-002"), EncodingType.CL100K_BASE) + OpenAIEmbeddings( + provider = this, + modelID = ModelID("text-embedding-ada-002"), + encodingType = EncodingType.CL100K_BASE + ) .autoCloseBind() } val DALLE_2 by lazy { - OpenAIImages(this, ModelID("dalle-2"), EncodingType.CL100K_BASE).autoCloseBind() + OpenAIImages( + provider = this, + modelID = ModelID("dalle-2"), + encodingType = EncodingType.CL100K_BASE + ) + .autoCloseBind() } @JvmField val DEFAULT_CHAT = GPT_3_5_TURBO_16K diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt index ed7498de8..eafa29a90 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt @@ -23,7 +23,8 @@ class OpenAICompletion( private val client = provider.defaultClient - override fun copy(modelID: ModelID) = OpenAICompletion(provider, modelID, contextLength, encodingType) + override fun copy(modelID: ModelID) = + OpenAICompletion(provider, modelID, contextLength, encodingType) @OptIn(LegacyOpenAI::class) override suspend fun createCompletion(request: CompletionRequest): CompletionResult { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt index 9c6518639..70484e9fa 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt @@ -27,7 +27,8 @@ class OpenAIFunChat( private val client = provider.defaultClient - override fun copy(modelID: ModelID) = OpenAIFunChat(provider, modelID, contextLength, encodingType) + override fun copy(modelID: ModelID) = + OpenAIFunChat(provider, modelID, contextLength, encodingType) override suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt index 8f4372314..e7137d0aa 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt @@ -16,14 +16,11 @@ interface OpenAIModel : LLM { fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = messages.sumOf { message -> countTokens(message.role.name) + - countTokens(message.content) + - tokensPerMessage + - tokensPerName + countTokens(message.content) + + tokensPerMessage + + tokensPerName } + 3 - return encoding.countTokensFromMessages( - tokensPerMessage = 5, - tokensPerName = 5 - ) + 10 + return encoding.countTokensFromMessages(tokensPerMessage = 5, tokensPerName = 5) + 10 } override fun truncateText(text: String, maxTokens: Int): String = diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt index 929811cf7..4a95c03f8 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt @@ -39,10 +39,11 @@ class Summarize( summarizeLargeText(text = input, summaryLength = summaryLength) private suspend fun summarizeChunk(chunk: String, summaryLength: SummaryLength): String { - val maxContextLength = when(val contextLength = model.contextLength) { - is MaxIoContextLength.Combined -> contextLength.total - is MaxIoContextLength.Fix -> contextLength.input - } - 1000 // magic padding for functions and memory + val maxContextLength = + when (val contextLength = model.contextLength) { + is MaxIoContextLength.Combined -> contextLength.total + is MaxIoContextLength.Fix -> contextLength.input + } - 1000 // magic padding for functions and memory val promptTokens: Int = encodingType.encoding.countTokens(chunk) logger.info { "📝 Summarizing chunk with prompt tokens $promptTokens for length $summaryLength" @@ -77,10 +78,11 @@ class Summarize( } private suspend fun chunkText(text: String): List { - val maxTokens = when(val contextLength = model.contextLength) { - is MaxIoContextLength.Combined -> contextLength.total - is MaxIoContextLength.Fix -> contextLength.input - } - 2000 // magic padding for functions and memory + val maxTokens = + when (val contextLength = model.contextLength) { + is MaxIoContextLength.Combined -> contextLength.total + is MaxIoContextLength.Fix -> contextLength.input + } - 2000 // magic padding for functions and memory val firstPart = model.truncateText(text, maxTokens) val remainingText = text.removePrefix(firstPart) From a4eef4bc51084105407bc46e959203af019d6b34 Mon Sep 17 00:00:00 2001 From: ron Date: Thu, 5 Oct 2023 11:09:19 +0200 Subject: [PATCH 07/19] first compiling version (no clean build) --- .../com/xebia/functional/xef/llm/Chat.kt | 1 + .../functional/xef/llm/ChatWithFunctions.kt | 1 + .../com/xebia/functional/xef/llm/Completion.kt | 1 - .../kotlin/com/xebia/functional/xef/llm/LLM.kt | 18 ++++++++++++++++++ .../kotlin/com/xebia/functional/xef/gcp/GCP.kt | 12 ++++++++++-- .../kotlin/com/xebia/functional/xef/sql/SQL.kt | 2 +- .../xef/reasoning/code/DiffSummary.kt | 2 +- .../xef/reasoning/text/summarize/Summarize.kt | 12 ++++++------ 8 files changed, 38 insertions(+), 11 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index 82187cba8..d35a5f122 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -13,6 +13,7 @@ interface Chat : LLM { val contextLength: MaxIoContextLength + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") override val maxContextLength get() = (contextLength as MaxIoContextLength.Combined).total diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index 825810f8b..142c270d5 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -24,6 +24,7 @@ interface ChatWithFunctions : LLM { val contextLength: MaxIoContextLength + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") override val maxContextLength get() = (contextLength as MaxIoContextLength.Combined).total diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index 359a565e7..a0fb9d7a2 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -6,6 +6,5 @@ import com.xebia.functional.xef.llm.models.text.CompletionResult interface Completion : LLM { val contextLength: MaxIoContextLength - suspend fun createCompletion(request: CompletionRequest): CompletionResult } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index bb98a5d21..8985f00d4 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -1,5 +1,6 @@ package com.xebia.functional.xef.llm +import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.Message @@ -9,6 +10,14 @@ import com.xebia.functional.xef.llm.models.chat.Message val modelID: ModelID + @Deprecated("intermediary solution, will be removed in future PR") + val modelType: ModelType + get() = when { + modelID.value.lowercase().startsWith("gpt") -> EncodingType.CL100K_BASE + modelID.value.lowercase() == "text-embedding-ada-002" -> EncodingType.CL100K_BASE + else -> EncodingType.P50K_BASE + }.let { ModelType(it) } + @Deprecated("use modelID.value instead", replaceWith = ReplaceWith("modelID.value")) val name get() = modelID.value @@ -31,3 +40,12 @@ import com.xebia.functional.xef.llm.models.chat.Message override fun close() = Unit } + +/** + * intermediary solution + */ +class ModelType( + val encodingType: EncodingType, +) { + val encoding get() = encodingType.encoding +} diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt index e95214f67..2213bee46 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt @@ -8,6 +8,7 @@ import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.gcp.models.GcpChat import com.xebia.functional.xef.gcp.models.GcpEmbeddings import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore @@ -47,8 +48,15 @@ class GCP(projectId: String? = null, location: VertexAIRegion? = null, token: St val defaultClient = GcpClient(config) - val CODECHAT by lazy { GcpChat(this, ModelID("codechat-bison@001")) } - val TEXT_EMBEDDING_GECKO by lazy { GcpEmbeddings(this, ModelID("textembedding-gecko")) } + val CODECHAT by lazy { GcpChat( + provider = this, + modelID = ModelID("codechat-bison@001"), + contextLength = MaxIoContextLength.Fix(6144, 1024), + ) } + val TEXT_EMBEDDING_GECKO by lazy { GcpEmbeddings( + provider = this, + modelID = ModelID("textembedding-gecko"), + ) } @JvmField val DEFAULT_CHAT = CODECHAT @JvmField val DEFAULT_EMBEDDING = TEXT_EMBEDDING_GECKO diff --git a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt index 486f1fedf..eb6ca1c18 100644 --- a/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt +++ b/integrations/sql/src/main/kotlin/com/xebia/functional/xef/sql/SQL.kt @@ -98,7 +98,7 @@ private class JDBCSQLImpl( val results = resultSet.toDocuments(prompt) logger.debug { "Found: ${results.size} records" } val splitter = TokenTextSplitter( - modelType = config.model.modelType, chunkSize = config.model.modelType.maxContextLength / 2, chunkOverlap = 10 + encodingType = config.model.modelType.encodingType, chunkSize = config.model.maxContextLength / 2, chunkOverlap = 10 ) val splitDocuments = splitter.splitDocuments(results) logger.debug { "Split into: ${splitDocuments.size} documents" } diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt index 0ee8f196d..f5e295257 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt @@ -66,7 +66,7 @@ class DiffSummary( "Could not read diff from url ${extracted.url}" } else { val content = diff.bodyAsText() - if (chat.modelType.encoding.countTokens(content) >= chat.modelType.maxContextLength / 2) { + if (chat.modelType.encoding.countTokens(content) >= chat.maxContextLength / 2) { val summary = summarize(diff.bodyAsText()) createPRDescription(summary) } else { diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt index 4a95c03f8..1010a5b2b 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt @@ -44,7 +44,7 @@ class Summarize( is MaxIoContextLength.Combined -> contextLength.total is MaxIoContextLength.Fix -> contextLength.input } - 1000 // magic padding for functions and memory - val promptTokens: Int = encodingType.encoding.countTokens(chunk) + val promptTokens: Int = model.modelType.encoding.countTokens(chunk) logger.info { "📝 Summarizing chunk with prompt tokens $promptTokens for length $summaryLength" } @@ -58,7 +58,7 @@ class Summarize( """| |Given the following text: |```text - |${encodingType.encoding.truncateText(chunk, remainingTokens)} + |${model.modelType.encoding.truncateText(chunk, remainingTokens)} |``` """ .trimMargin() @@ -72,7 +72,7 @@ class Summarize( } return model.promptMessage(messages, scope).also { - val tokens: Int = encodingType.encoding.countTokens(it) + val tokens: Int = model.modelType.encoding.countTokens(it) logger.info { "📝 Summarized chunk in tokens: $tokens" } } } @@ -83,7 +83,7 @@ class Summarize( is MaxIoContextLength.Combined -> contextLength.total is MaxIoContextLength.Fix -> contextLength.input } - 2000 // magic padding for functions and memory - val firstPart = model.truncateText(text, maxTokens) + val firstPart = model.modelType.encoding.truncateText(text, maxTokens) val remainingText = text.removePrefix(firstPart) return if (remainingText.isNotEmpty()) { @@ -94,7 +94,7 @@ class Summarize( } tailrec suspend fun summarizeLargeText(text: String, summaryLength: SummaryLength): String { - val tokens = encodingType.encoding.countTokens(text) + val tokens = model.modelType.encoding.countTokens(text) logger.info { "📚 Summarizing large text of tokens ${tokens} to approximately $summaryLength tokens" } @@ -117,7 +117,7 @@ class Summarize( // Join the chunk summaries into one text val joinedSummaries = chunkSummaries.joinToString(" ") - val joinedSummariesTokens = encodingType.encoding.countTokens(joinedSummaries) + val joinedSummariesTokens = model.modelType.encoding.countTokens(joinedSummaries) // Resummarize the joined summaries if it is longer than summaryLength return if ( From 3aa8be600b837daab50f19deae30ca9f2f62cfbc Mon Sep 17 00:00:00 2001 From: ron Date: Thu, 5 Oct 2023 11:21:13 +0200 Subject: [PATCH 08/19] spotless --- .../xebia/functional/xef/llm/Completion.kt | 1 + .../com/xebia/functional/xef/llm/LLM.kt | 18 +++++++-------- .../com/xebia/functional/xef/gcp/GCP.kt | 22 +++++++++++-------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index a0fb9d7a2..359a565e7 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -6,5 +6,6 @@ import com.xebia.functional.xef.llm.models.text.CompletionResult interface Completion : LLM { val contextLength: MaxIoContextLength + suspend fun createCompletion(request: CompletionRequest): CompletionResult } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 8985f00d4..241fb8b8a 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -12,11 +12,12 @@ import com.xebia.functional.xef.llm.models.chat.Message @Deprecated("intermediary solution, will be removed in future PR") val modelType: ModelType - get() = when { - modelID.value.lowercase().startsWith("gpt") -> EncodingType.CL100K_BASE - modelID.value.lowercase() == "text-embedding-ada-002" -> EncodingType.CL100K_BASE - else -> EncodingType.P50K_BASE - }.let { ModelType(it) } + get() = + when { + modelID.value.lowercase().startsWith("gpt") -> EncodingType.CL100K_BASE + modelID.value.lowercase() == "text-embedding-ada-002" -> EncodingType.CL100K_BASE + else -> EncodingType.P50K_BASE + }.let { ModelType(it) } @Deprecated("use modelID.value instead", replaceWith = ReplaceWith("modelID.value")) val name @@ -41,11 +42,10 @@ import com.xebia.functional.xef.llm.models.chat.Message override fun close() = Unit } -/** - * intermediary solution - */ +/** intermediary solution */ class ModelType( val encodingType: EncodingType, ) { - val encoding get() = encodingType.encoding + val encoding + get() = encodingType.encoding } diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt index 2213bee46..b7d3c90e0 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/GCP.kt @@ -48,15 +48,19 @@ class GCP(projectId: String? = null, location: VertexAIRegion? = null, token: St val defaultClient = GcpClient(config) - val CODECHAT by lazy { GcpChat( - provider = this, - modelID = ModelID("codechat-bison@001"), - contextLength = MaxIoContextLength.Fix(6144, 1024), - ) } - val TEXT_EMBEDDING_GECKO by lazy { GcpEmbeddings( - provider = this, - modelID = ModelID("textembedding-gecko"), - ) } + val CODECHAT by lazy { + GcpChat( + provider = this, + modelID = ModelID("codechat-bison@001"), + contextLength = MaxIoContextLength.Fix(6144, 1024), + ) + } + val TEXT_EMBEDDING_GECKO by lazy { + GcpEmbeddings( + provider = this, + modelID = ModelID("textembedding-gecko"), + ) + } @JvmField val DEFAULT_CHAT = CODECHAT @JvmField val DEFAULT_EMBEDDING = TEXT_EMBEDDING_GECKO From a267c8857e71a71c554f361eb31436433a31eca2 Mon Sep 17 00:00:00 2001 From: ron Date: Thu, 5 Oct 2023 11:59:32 +0200 Subject: [PATCH 09/19] pls compile compadre --- .../xef/conversation/ConversationSpec.kt | 17 ++++++++--------- .../functional/xef/data/TestFunctionsModel.kt | 7 +++++-- .../com/xebia/functional/xef/data/TestModel.kt | 7 +++++-- .../xef/store/CombinedVectorStoreSpec.kt | 8 ++++---- .../xef/store/LocalVectorStoreSpec.kt | 6 +++--- .../src/test/kotlin/xef/PGVectorStoreSpec.kt | 2 ++ 6 files changed, 27 insertions(+), 20 deletions(-) diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index 2f2994dbf..ffea10427 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -1,6 +1,7 @@ package com.xebia.functional.xef.conversation import com.xebia.functional.xef.data.* +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.chat.Role import com.xebia.functional.xef.prompt.Prompt @@ -23,7 +24,7 @@ class ConversationSpec : "memories should have the correct size in the vector store" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) @@ -50,7 +51,7 @@ class ConversationSpec : val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) val vectorStore = scope.store - val modelAda = TestModel(modelType = ModelType.ADA, responses = messages) + val modelAda = TestModel(responses = messages) val totalTokens = modelAda.tokensFromMessages( @@ -88,7 +89,7 @@ class ConversationSpec : val vectorStore = scope.store val modelGPTTurbo16K = - TestModel(modelType = ModelType.GPT_3_5_TURBO_16_K, responses = messages) + TestModel(contextLength = MaxIoContextLength.Combined(16388), responses = messages) val totalTokens = modelGPTTurbo16K.tokensFromMessages( @@ -123,8 +124,7 @@ class ConversationSpec : val conversationId = ConversationId(UUID.generateUUID().toString()) val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) - val model = - TestFunctionsModel(modelType = ModelType.GPT_3_5_TURBO_FUNCTIONS, responses = message) + val model = TestFunctionsModel(responses = message) val response: Answer = model.prompt(prompt = Prompt(question), scope = scope, serializer = Answer.serializer()) @@ -147,8 +147,7 @@ class ConversationSpec : val conversationId = ConversationId(UUID.generateUUID().toString()) val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) - val model = - TestFunctionsModel(modelType = ModelType.GPT_3_5_TURBO_FUNCTIONS, responses = message) + val model = TestFunctionsModel(responses = message) val response: Answer = model.prompt( @@ -167,7 +166,7 @@ class ConversationSpec : "the scope's store should contains all the messages" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val scope = Conversation(LocalVectorStore(TestEmbeddings()), conversationId = conversationId) @@ -213,7 +212,7 @@ class ConversationSpec : "when using 2 different scopes with the same conversationId, the index of the memories should be in order" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt index c90b741f3..cc29e535b 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt @@ -2,6 +2,7 @@ package com.xebia.functional.xef.data import com.xebia.functional.xef.llm.ChatWithFunctions import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest @@ -12,13 +13,15 @@ import com.xebia.functional.xef.llm.models.usage.Usage import kotlinx.coroutines.flow.Flow class TestFunctionsModel( - override val modelID: ModelID, + override val contextLength: MaxIoContextLength = MaxIoContextLength.Combined(Int.MAX_VALUE), val responses: Map = emptyMap(), ) : ChatWithFunctions, Embeddings, AutoCloseable { + override val modelID = ModelID("test-model") + var requests: MutableList = mutableListOf() - override fun copy(modelID: ModelID) = TestFunctionsModel(modelID, responses) + override fun copy(modelID: ModelID) = TestFunctionsModel(contextLength, responses) override fun tokensFromMessages(messages: List): Int { return messages.sumOf { it.content.length } diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt index c947b14a0..8f5d4ad87 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt @@ -2,6 +2,7 @@ package com.xebia.functional.xef.data import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Embeddings +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.embeddings.EmbeddingRequest @@ -10,13 +11,15 @@ import com.xebia.functional.xef.llm.models.usage.Usage import kotlinx.coroutines.flow.Flow class TestModel( - override val modelID: ModelID, + override val contextLength: MaxIoContextLength = MaxIoContextLength.Combined(Int.MAX_VALUE), val responses: Map = emptyMap(), ) : Chat, Embeddings, AutoCloseable { + override val modelID = ModelID("test-model") + var requests: MutableList = mutableListOf() - override fun copy(modelType: ModelID) = TestModel(modelType, responses) + override fun copy(modelType: ModelID) = TestModel(contextLength, responses) override suspend fun createChatCompletion( request: ChatCompletionRequest diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt index 63a276331..eac5a9271 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/CombinedVectorStoreSpec.kt @@ -8,7 +8,7 @@ import io.kotest.matchers.shouldBe class CombinedVectorStoreSpec : StringSpec({ "memories function should return all of messages combined in the right order" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val memoryData = MemoryData() @@ -26,7 +26,7 @@ class CombinedVectorStoreSpec : } "memories function should return the last n combined messages in the right order" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val memoryData = MemoryData() @@ -52,7 +52,7 @@ class CombinedVectorStoreSpec : } "memories function should return the messages with common conversation id combined in the right order" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val memoryData = MemoryData() @@ -80,7 +80,7 @@ class CombinedVectorStoreSpec : } "adding messages to a combined vector store" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val memoryData = MemoryData() diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt index fc4069cf6..44bc13d52 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/store/LocalVectorStoreSpec.kt @@ -8,7 +8,7 @@ import io.kotest.matchers.shouldBe class LocalVectorStoreSpec : StringSpec({ "memories function should return all of messages in the right order when the limit is greater than the number of stored messages" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val localVectorStore = LocalVectorStore(TestEmbeddings()) val memoryData = MemoryData() @@ -28,7 +28,7 @@ class LocalVectorStoreSpec : } "memories function should return the last n messages in the right order" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val localVectorStore = LocalVectorStore(TestEmbeddings()) val memoryData = MemoryData() @@ -48,7 +48,7 @@ class LocalVectorStoreSpec : } "memories function should return the last n messages in the right order for a specific conversation id" { - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val localVectorStore = LocalVectorStore(TestEmbeddings()) val firstId = ConversationId("first-id") diff --git a/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt b/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt index 387837cdc..a0bdc2060 100644 --- a/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt +++ b/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt @@ -3,6 +3,7 @@ package xef import com.xebia.functional.xef.llm.Chat import com.xebia.functional.xef.llm.Embeddings import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.llm.models.embeddings.Embedding @@ -112,6 +113,7 @@ class PGVectorStoreSpec : class TestLLM : Chat, AutoCloseable { override val modelID = ModelID("test-llm") + override val contextLength = MaxIoContextLength.Combined(Integer.MAX_VALUE) override fun copy(modelID: ModelID) = TestLLM() From f0075a4b2db139a94552f8cab8624d6f9a7e7251 Mon Sep 17 00:00:00 2001 From: ron Date: Thu, 5 Oct 2023 14:51:37 +0200 Subject: [PATCH 10/19] small changes --- core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt | 2 +- .../com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 241fb8b8a..8512df7fc 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -42,7 +42,7 @@ import com.xebia.functional.xef.llm.models.chat.Message override fun close() = Unit } -/** intermediary solution */ +@Deprecated("intermediary solution; see #405") class ModelType( val encodingType: EncodingType, ) { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 89fc47531..754d7dd9b 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -32,7 +32,6 @@ class OpenAI(internal var token: String? = null, internal var host: String? = nu AutoCloseable, AutoClose by autoClose() { private fun openAITokenFromEnv(): String { - OpenAIHost.OpenAI.baseUrl return getenv(KEY_ENV_VAR) ?: throw AIError.Env.OpenAI(nonEmptyListOf("missing $KEY_ENV_VAR env var")) } From 9f2329594c4401b61e18265bb70a6692e77a1839 Mon Sep 17 00:00:00 2001 From: ron Date: Thu, 5 Oct 2023 15:15:48 +0200 Subject: [PATCH 11/19] small changes --- .../kotlin/com/xebia/functional/xef/llm/Chat.kt | 6 +++++- .../functional/xef/llm/ChatWithFunctions.kt | 6 +++++- .../xef/reasoning/text/summarize/Summarize.kt | 16 ++++------------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index d35a5f122..55e5dbda0 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -15,7 +15,11 @@ interface Chat : LLM { @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") override val maxContextLength - get() = (contextLength as MaxIoContextLength.Combined).total + get() = + (contextLength as? MaxIoContextLength.Combined)?.total + ?: error( + "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" + ) suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index 142c270d5..2d3086cfe 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -26,7 +26,11 @@ interface ChatWithFunctions : LLM { @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") override val maxContextLength - get() = (contextLength as MaxIoContextLength.Combined).total + get() = + (contextLength as? MaxIoContextLength.Combined)?.total + ?: error( + "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" + ) suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt index 1010a5b2b..2c628d129 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt @@ -4,7 +4,6 @@ import arrow.fx.coroutines.parMap import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.llm.Chat -import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistantSteps import com.xebia.functional.xef.prompt.templates.system @@ -39,11 +38,8 @@ class Summarize( summarizeLargeText(text = input, summaryLength = summaryLength) private suspend fun summarizeChunk(chunk: String, summaryLength: SummaryLength): String { - val maxContextLength = - when (val contextLength = model.contextLength) { - is MaxIoContextLength.Combined -> contextLength.total - is MaxIoContextLength.Fix -> contextLength.input - } - 1000 // magic padding for functions and memory + val maxContextLength: Int = + model.maxContextLength - 1000 // magic padding for functions and memory val promptTokens: Int = model.modelType.encoding.countTokens(chunk) logger.info { "📝 Summarizing chunk with prompt tokens $promptTokens for length $summaryLength" @@ -77,12 +73,8 @@ class Summarize( } } - private suspend fun chunkText(text: String): List { - val maxTokens = - when (val contextLength = model.contextLength) { - is MaxIoContextLength.Combined -> contextLength.total - is MaxIoContextLength.Fix -> contextLength.input - } - 2000 // magic padding for functions and memory + private fun chunkText(text: String): List { + val maxTokens = model.maxContextLength - 2000 // magic padding for functions and memory val firstPart = model.modelType.encoding.truncateText(text, maxTokens) val remainingText = text.removePrefix(firstPart) From a2512259263f905bf41f225133da9d0b2469e5e7 Mon Sep 17 00:00:00 2001 From: ron Date: Fri, 24 Nov 2023 19:45:52 +0100 Subject: [PATCH 12/19] fixed stuff after merge --- .../kotlin/com/xebia/functional/xef/llm/Embeddings.kt | 2 +- .../jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt | 0 .../xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt | 0 .../xebia/functional/xef/conversation/llm/openai/OpenAI.kt | 4 ++-- 4 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt delete mode 100644 gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt index e11e28bba..d9a48971b 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Embeddings.kt @@ -19,7 +19,7 @@ interface Embeddings : LLM { texts .chunked(chunkSize ?: 400) .parMap { - createEmbeddings(EmbeddingRequest(modelType.name, it, requestConfig.user.id)).data + createEmbeddings(EmbeddingRequest(modelID.value, it, requestConfig.user.id)).data } .flatten() diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/GPT4All.kt deleted file mode 100644 index e69de29bb..000000000 diff --git a/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt b/gpt4all-kotlin/src/jvmMain/kotlin/com/xebia/functional/gpt4all/HuggingFaceLocalEmbeddings.kt deleted file mode 100644 index e69de29bb..000000000 diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index f806f4b75..5639d8a80 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -260,7 +260,7 @@ class OpenAI( either { // TODO: impl of abstract provider function ensure(modelExists(modelId)) { "model $modelId not found" } @Suppress("UNCHECKED_CAST") - baseModel.copy(ModelType.FineTunedModel(modelId, baseModel = baseModel.modelType)) as? T + baseModel.copy(modelID = ModelID(modelId)) as? T ?: error("${baseModel::class} does not follow contract to return the most specific type") } @@ -276,7 +276,7 @@ class OpenAI( ensureNotNull(job) { "job $fineTuningJobId not found" } val fineTunedModel = job.fineTunedModel ensureNotNull(fineTunedModel) { "fine tuned model not available, status ${job.status}" } - ensure(baseModel.modelType.name == job.model.id) { + ensure(baseModel.modelID.value == job.model.id) { "base model instance does not match the job's base model" } spawnModel(fineTunedModel.id, baseModel).bind() From 96fe57c28ecd0807d1c3f2ed106305dc8a001c76 Mon Sep 17 00:00:00 2001 From: Intex32 Date: Fri, 24 Nov 2023 19:06:24 +0000 Subject: [PATCH 13/19] Apply spotless formatting --- .../com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt index 5639d8a80..af3917d95 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/OpenAI.kt @@ -20,9 +20,9 @@ import com.xebia.functional.xef.conversation.autoClose import com.xebia.functional.xef.conversation.llm.openai.models.* import com.xebia.functional.xef.env.getenv import com.xebia.functional.xef.llm.LLM -import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID +import com.xebia.functional.xef.metrics.Metric import com.xebia.functional.xef.store.LocalVectorStore import com.xebia.functional.xef.store.VectorStore import kotlin.jvm.JvmField From 7a628e03912e75e3788ec1da7bfc6b97cd20adbe Mon Sep 17 00:00:00 2001 From: ron Date: Fri, 24 Nov 2023 21:31:32 +0100 Subject: [PATCH 14/19] many small changes; introduces BaseChat --- .../com/xebia/functional/xef/llm/BaseChat.kt | 27 +++++++++++++++++ .../com/xebia/functional/xef/llm/Chat.kt | 14 ++------- .../functional/xef/llm/ChatWithFunctions.kt | 13 +------- .../xebia/functional/xef/llm/Completion.kt | 8 +++++ .../com/xebia/functional/xef/llm/LLM.kt | 30 +++++-------------- .../functional/xef/llm/PromptCalculator.kt | 6 ++-- .../functional/xef/gcp/models/GcpChat.kt | 12 ++++++++ .../xef/gcp/models/GcpCompletion.kt | 9 ++++++ .../llm/openai/models/OpenAIChat.kt | 19 ++++++++++++ .../llm/openai/models/OpenAICompletion.kt | 9 ++++++ .../llm/openai/models/OpenAIFunChat.kt | 19 ++++++++++++ .../llm/openai/models/OpenAIModel.kt | 13 -------- .../xef/reasoning/code/DiffSummary.kt | 2 +- .../xef/reasoning/text/summarize/Summarize.kt | 12 ++++---- 14 files changed, 124 insertions(+), 69 deletions(-) create mode 100644 core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt new file mode 100644 index 000000000..157eca3bf --- /dev/null +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt @@ -0,0 +1,27 @@ +package com.xebia.functional.xef.llm + +import com.xebia.functional.xef.llm.models.MaxIoContextLength +import com.xebia.functional.xef.llm.models.chat.Message + +interface BaseChat : LLM { + + val contextLength: MaxIoContextLength + + @Deprecated("will be removed from LLM in favor of abstracting former ModelType, use contextLength instead") + override val maxContextLength + get() = + (contextLength as? MaxIoContextLength.Combined)?.total + ?: error( + "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" + ) + + @Suppress("OVERRIDE_DEPRECATION") + override fun countTokens(text: String): Int + + @Suppress("OVERRIDE_DEPRECATION") + override fun truncateText(text: String, maxTokens: Int): String + + @Suppress("OVERRIDE_DEPRECATION") + override fun tokensFromMessages(messages: List): Int + +} diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index b046cde42..e4c50f7d4 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -3,23 +3,12 @@ package com.xebia.functional.xef.llm import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation -import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.chat.* import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistant import kotlinx.coroutines.flow.* -interface Chat : LLM { - - val contextLength: MaxIoContextLength - - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") - override val maxContextLength - get() = - (contextLength as? MaxIoContextLength.Combined)?.total - ?: error( - "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" - ) +interface Chat : BaseChat { suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse @@ -83,4 +72,5 @@ interface Chat : LLM { ) .mapNotNull { it.message?.content } } + } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt index 9b79375f1..8b9ad3515 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/ChatWithFunctions.kt @@ -6,7 +6,6 @@ import arrow.core.raise.catch import com.xebia.functional.xef.AIError import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation -import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.chat.ChatCompletionChunk import com.xebia.functional.xef.llm.models.chat.ChatCompletionResponseWithFunctions import com.xebia.functional.xef.llm.models.functions.CFunction @@ -20,17 +19,7 @@ import kotlinx.serialization.KSerializer import kotlinx.serialization.descriptors.* import kotlinx.serialization.json.* -interface ChatWithFunctions : LLM { - - val contextLength: MaxIoContextLength - - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") - override val maxContextLength - get() = - (contextLength as? MaxIoContextLength.Combined)?.total - ?: error( - "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" - ) +interface ChatWithFunctions : BaseChat { suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index 359a565e7..f2833c850 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -1,6 +1,9 @@ package com.xebia.functional.xef.llm +import com.xebia.functional.tokenizer.Encoding +import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.llm.models.MaxIoContextLength +import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult @@ -8,4 +11,9 @@ interface Completion : LLM { val contextLength: MaxIoContextLength suspend fun createCompletion(request: CompletionRequest): CompletionResult + + override fun countTokens(text: String): Int + + override fun truncateText(text: String, maxTokens: Int): String + } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 8512df7fc..247d6ff68 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -10,42 +10,28 @@ import com.xebia.functional.xef.llm.models.chat.Message val modelID: ModelID - @Deprecated("intermediary solution, will be removed in future PR") - val modelType: ModelType - get() = - when { - modelID.value.lowercase().startsWith("gpt") -> EncodingType.CL100K_BASE - modelID.value.lowercase() == "text-embedding-ada-002" -> EncodingType.CL100K_BASE - else -> EncodingType.P50K_BASE - }.let { ModelType(it) } - @Deprecated("use modelID.value instead", replaceWith = ReplaceWith("modelID.value")) val name get() = modelID.value /** - * Copies this instance and uses [modelType] for [LLM.modelType]. Has to return the most specific - * type of this instance! + * Copies this instance and uses [modelID] for the new instances' [LLM.modelID]. + * Has to return the most specific type of this instance! */ fun copy(modelID: ModelID): LLM - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") fun tokensFromMessages(messages: List): Int = TODO() // intermediary - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") + fun countTokens(text: String): Int = TODO() // intermediary + + @Deprecated("will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") fun truncateText(text: String, maxTokens: Int): String = TODO() // intermediary - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType") + @Deprecated("will be removed from LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") val maxContextLength: Int get() = TODO() override fun close() = Unit } - -@Deprecated("intermediary solution; see #405") -class ModelType( - val encodingType: EncodingType, -) { - val encoding - get() = encodingType.encoding -} diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt index 5394fdb8e..3dc27fcad 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt @@ -14,7 +14,7 @@ internal object PromptCalculator { suspend fun adaptPromptToConversationAndModel( prompt: Prompt, scope: Conversation, - llm: LLM + llm: BaseChat ): Prompt = when (prompt.configuration.messagePolicy.addMessagesFromConversation) { MessagesFromHistory.ALL -> adaptPromptFromConversation(prompt, scope, llm) @@ -24,7 +24,7 @@ internal object PromptCalculator { private suspend fun adaptPromptFromConversation( prompt: Prompt, scope: Conversation, - llm: LLM + llm: BaseChat ): Prompt { // calculate tokens for history and context @@ -69,7 +69,7 @@ internal object PromptCalculator { memories.map { it.content } private fun calculateMessagesFromHistory( - llm: LLM, + llm: BaseChat, memories: List, maxHistoryTokens: Int ) = diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt index 67ff5b12d..0d040740f 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpChat.kt @@ -84,4 +84,16 @@ class GcpChat( } return "$messages\n### Response:" } + + override fun countTokens(text: String): Int { + TODO("Not yet implemented") + } + + override fun truncateText(text: String, maxTokens: Int): String { + TODO("Not yet implemented") + } + + override fun tokensFromMessages(messages: List): Int { + TODO("Not yet implemented") + } } diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt index a9c574787..f1ef88e84 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt @@ -40,4 +40,13 @@ class GcpCompletion( Usage.ZERO, // TODO: token usage - no information about usage provided by GCP codechat model ) } + + override fun countTokens(text: String): Int { + TODO("Not yet implemented") + } + + override fun truncateText(text: String, maxTokens: Int): String { + TODO("Not yet implemented") + } + } diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt index bee199434..17ba3e2a7 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt @@ -4,7 +4,9 @@ import com.aallam.openai.api.chat.ChatChoice import com.aallam.openai.api.chat.ChatMessage import com.aallam.openai.api.chat.chatCompletionRequest import com.aallam.openai.api.model.ModelId +import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.tokenizer.EncodingType +import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.conversation.llm.openai.toOpenAI @@ -26,6 +28,23 @@ class OpenAIChat( override fun copy(modelID: ModelID) = OpenAIChat(provider, modelID, contextLength, encodingType) + override fun countTokens(text: String): Int = + encoding.countTokens(text) + + override fun truncateText(text: String, maxTokens: Int): String = + encoding.truncateText(text, maxTokens) + + override fun tokensFromMessages(messages: List): Int { // intermediary solution + fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = + messages.sumOf { message -> + countTokens(message.role.name) + + countTokens(message.content) + + tokensPerMessage + + tokensPerName + } + 3 + return encoding.countTokensFromMessages(tokensPerMessage = 5, tokensPerName = 5) + 10 + } + override suspend fun createChatCompletion( request: ChatCompletionRequest ): ChatCompletionResponse { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt index eafa29a90..2804ab44c 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt @@ -4,12 +4,15 @@ import com.aallam.openai.api.LegacyOpenAI import com.aallam.openai.api.completion.Choice import com.aallam.openai.api.completion.completionRequest import com.aallam.openai.api.model.ModelId +import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.tokenizer.EncodingType +import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.llm.Completion import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID +import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult @@ -26,6 +29,12 @@ class OpenAICompletion( override fun copy(modelID: ModelID) = OpenAICompletion(provider, modelID, contextLength, encodingType) + override fun countTokens(text: String): Int = + encoding.countTokens(text) + + override fun truncateText(text: String, maxTokens: Int): String = + encoding.truncateText(text, maxTokens) + @OptIn(LegacyOpenAI::class) override suspend fun createCompletion(request: CompletionRequest): CompletionResult { fun toInternal(it: Choice): CompletionChoice = diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt index 7a1687d78..24196f156 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt @@ -2,7 +2,9 @@ package com.xebia.functional.xef.conversation.llm.openai.models import com.aallam.openai.api.chat.* import com.aallam.openai.api.model.ModelId +import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.tokenizer.EncodingType +import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.conversation.llm.openai.toOpenAI @@ -29,6 +31,23 @@ class OpenAIFunChat( override fun copy(modelID: ModelID) = OpenAIFunChat(provider, modelID, contextLength, encodingType) + override fun countTokens(text: String): Int = + encoding.countTokens(text) + + override fun truncateText(text: String, maxTokens: Int): String = + encoding.truncateText(text, maxTokens) + + override fun tokensFromMessages(messages: List): Int { // intermediary solution + fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = + messages.sumOf { message -> + countTokens(message.role.name) + + countTokens(message.content) + + tokensPerMessage + + tokensPerName + } + 3 + return encoding.countTokensFromMessages(tokensPerMessage = 5, tokensPerName = 5) + 10 + } + override suspend fun createChatCompletionWithFunctions( request: FunChatCompletionRequest ): ChatCompletionResponseWithFunctions { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt index e7137d0aa..316f70122 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt @@ -12,17 +12,4 @@ interface OpenAIModel : LLM { val encoding get() = encodingType.encoding - override fun tokensFromMessages(messages: List): Int { // intermediary solution - fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = - messages.sumOf { message -> - countTokens(message.role.name) + - countTokens(message.content) + - tokensPerMessage + - tokensPerName - } + 3 - return encoding.countTokensFromMessages(tokensPerMessage = 5, tokensPerName = 5) + 10 - } - - override fun truncateText(text: String, maxTokens: Int): String = - encoding.truncateText(text, maxTokens) } diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt index f5e295257..c1f07a1e1 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/code/DiffSummary.kt @@ -66,7 +66,7 @@ class DiffSummary( "Could not read diff from url ${extracted.url}" } else { val content = diff.bodyAsText() - if (chat.modelType.encoding.countTokens(content) >= chat.maxContextLength / 2) { + if (chat.countTokens(content) >= chat.maxContextLength / 2) { val summary = summarize(diff.bodyAsText()) createPRDescription(summary) } else { diff --git a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt index 2c628d129..59d5c6db6 100644 --- a/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt +++ b/reasoning/src/commonMain/kotlin/com/xebia/functional/xef/reasoning/text/summarize/Summarize.kt @@ -40,7 +40,7 @@ class Summarize( private suspend fun summarizeChunk(chunk: String, summaryLength: SummaryLength): String { val maxContextLength: Int = model.maxContextLength - 1000 // magic padding for functions and memory - val promptTokens: Int = model.modelType.encoding.countTokens(chunk) + val promptTokens: Int = model.countTokens(chunk) logger.info { "📝 Summarizing chunk with prompt tokens $promptTokens for length $summaryLength" } @@ -54,7 +54,7 @@ class Summarize( """| |Given the following text: |```text - |${model.modelType.encoding.truncateText(chunk, remainingTokens)} + |${model.truncateText(chunk, remainingTokens)} |``` """ .trimMargin() @@ -68,14 +68,14 @@ class Summarize( } return model.promptMessage(messages, scope).also { - val tokens: Int = model.modelType.encoding.countTokens(it) + val tokens: Int = model.countTokens(it) logger.info { "📝 Summarized chunk in tokens: $tokens" } } } private fun chunkText(text: String): List { val maxTokens = model.maxContextLength - 2000 // magic padding for functions and memory - val firstPart = model.modelType.encoding.truncateText(text, maxTokens) + val firstPart = model.truncateText(text, maxTokens) val remainingText = text.removePrefix(firstPart) return if (remainingText.isNotEmpty()) { @@ -86,7 +86,7 @@ class Summarize( } tailrec suspend fun summarizeLargeText(text: String, summaryLength: SummaryLength): String { - val tokens = model.modelType.encoding.countTokens(text) + val tokens = model.countTokens(text) logger.info { "📚 Summarizing large text of tokens ${tokens} to approximately $summaryLength tokens" } @@ -109,7 +109,7 @@ class Summarize( // Join the chunk summaries into one text val joinedSummaries = chunkSummaries.joinToString(" ") - val joinedSummariesTokens = model.modelType.encoding.countTokens(joinedSummaries) + val joinedSummariesTokens = model.countTokens(joinedSummaries) // Resummarize the joined summaries if it is longer than summaryLength return if ( From c6b56e28c71daa25675b646c684f6d1641aaf1d9 Mon Sep 17 00:00:00 2001 From: Intex32 Date: Fri, 24 Nov 2023 20:32:55 +0000 Subject: [PATCH 15/19] Apply spotless formatting --- .../com/xebia/functional/xef/llm/BaseChat.kt | 14 ++++++------- .../com/xebia/functional/xef/llm/Chat.kt | 1 - .../xebia/functional/xef/llm/Completion.kt | 4 ---- .../com/xebia/functional/xef/llm/LLM.kt | 21 ++++++++++++------- .../xef/gcp/models/GcpCompletion.kt | 1 - .../llm/openai/models/OpenAIChat.kt | 9 ++++---- .../llm/openai/models/OpenAICompletion.kt | 5 +---- .../llm/openai/models/OpenAIFunChat.kt | 9 ++++---- .../llm/openai/models/OpenAIModel.kt | 4 ---- 9 files changed, 29 insertions(+), 39 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt index 157eca3bf..216339314 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt @@ -7,7 +7,9 @@ interface BaseChat : LLM { val contextLength: MaxIoContextLength - @Deprecated("will be removed from LLM in favor of abstracting former ModelType, use contextLength instead") + @Deprecated( + "will be removed from LLM in favor of abstracting former ModelType, use contextLength instead" + ) override val maxContextLength get() = (contextLength as? MaxIoContextLength.Combined)?.total @@ -15,13 +17,9 @@ interface BaseChat : LLM { "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" ) - @Suppress("OVERRIDE_DEPRECATION") - override fun countTokens(text: String): Int + @Suppress("OVERRIDE_DEPRECATION") override fun countTokens(text: String): Int - @Suppress("OVERRIDE_DEPRECATION") - override fun truncateText(text: String, maxTokens: Int): String - - @Suppress("OVERRIDE_DEPRECATION") - override fun tokensFromMessages(messages: List): Int + @Suppress("OVERRIDE_DEPRECATION") override fun truncateText(text: String, maxTokens: Int): String + @Suppress("OVERRIDE_DEPRECATION") override fun tokensFromMessages(messages: List): Int } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt index e4c50f7d4..05474a10f 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Chat.kt @@ -72,5 +72,4 @@ interface Chat : BaseChat { ) .mapNotNull { it.message?.content } } - } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index f2833c850..898679872 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -1,9 +1,6 @@ package com.xebia.functional.xef.llm -import com.xebia.functional.tokenizer.Encoding -import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.llm.models.MaxIoContextLength -import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult @@ -15,5 +12,4 @@ interface Completion : LLM { override fun countTokens(text: String): Int override fun truncateText(text: String, maxTokens: Int): String - } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 247d6ff68..8ef60ec4d 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -1,6 +1,5 @@ package com.xebia.functional.xef.llm -import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.llm.models.ModelID import com.xebia.functional.xef.llm.models.chat.Message @@ -15,21 +14,29 @@ import com.xebia.functional.xef.llm.models.chat.Message get() = modelID.value /** - * Copies this instance and uses [modelID] for the new instances' [LLM.modelID]. - * Has to return the most specific type of this instance! + * Copies this instance and uses [modelID] for the new instances' [LLM.modelID]. Has to return the + * most specific type of this instance! */ fun copy(modelID: ModelID): LLM - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") + @Deprecated( + "will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" + ) fun tokensFromMessages(messages: List): Int = TODO() // intermediary - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") + @Deprecated( + "will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" + ) fun countTokens(text: String): Int = TODO() // intermediary - @Deprecated("will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") + @Deprecated( + "will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" + ) fun truncateText(text: String, maxTokens: Int): String = TODO() // intermediary - @Deprecated("will be removed from LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs") + @Deprecated( + "will be removed from LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" + ) val maxContextLength: Int get() = TODO() diff --git a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt index f1ef88e84..925402f5a 100644 --- a/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt +++ b/integrations/gcp/src/commonMain/kotlin/com/xebia/functional/xef/gcp/models/GcpCompletion.kt @@ -48,5 +48,4 @@ class GcpCompletion( override fun truncateText(text: String, maxTokens: Int): String { TODO("Not yet implemented") } - } diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt index 17ba3e2a7..41afcd510 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIChat.kt @@ -28,8 +28,7 @@ class OpenAIChat( override fun copy(modelID: ModelID) = OpenAIChat(provider, modelID, contextLength, encodingType) - override fun countTokens(text: String): Int = - encoding.countTokens(text) + override fun countTokens(text: String): Int = encoding.countTokens(text) override fun truncateText(text: String, maxTokens: Int): String = encoding.truncateText(text, maxTokens) @@ -38,9 +37,9 @@ class OpenAIChat( fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = messages.sumOf { message -> countTokens(message.role.name) + - countTokens(message.content) + - tokensPerMessage + - tokensPerName + countTokens(message.content) + + tokensPerMessage + + tokensPerName } + 3 return encoding.countTokensFromMessages(tokensPerMessage = 5, tokensPerName = 5) + 10 } diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt index 2804ab44c..f6dd47f42 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAICompletion.kt @@ -4,7 +4,6 @@ import com.aallam.openai.api.LegacyOpenAI import com.aallam.openai.api.completion.Choice import com.aallam.openai.api.completion.completionRequest import com.aallam.openai.api.model.ModelId -import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.conversation.llm.openai.OpenAI @@ -12,7 +11,6 @@ import com.xebia.functional.xef.conversation.llm.openai.toInternal import com.xebia.functional.xef.llm.Completion import com.xebia.functional.xef.llm.models.MaxIoContextLength import com.xebia.functional.xef.llm.models.ModelID -import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.text.CompletionChoice import com.xebia.functional.xef.llm.models.text.CompletionRequest import com.xebia.functional.xef.llm.models.text.CompletionResult @@ -29,8 +27,7 @@ class OpenAICompletion( override fun copy(modelID: ModelID) = OpenAICompletion(provider, modelID, contextLength, encodingType) - override fun countTokens(text: String): Int = - encoding.countTokens(text) + override fun countTokens(text: String): Int = encoding.countTokens(text) override fun truncateText(text: String, maxTokens: Int): String = encoding.truncateText(text, maxTokens) diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt index 24196f156..5394a1bda 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIFunChat.kt @@ -31,8 +31,7 @@ class OpenAIFunChat( override fun copy(modelID: ModelID) = OpenAIFunChat(provider, modelID, contextLength, encodingType) - override fun countTokens(text: String): Int = - encoding.countTokens(text) + override fun countTokens(text: String): Int = encoding.countTokens(text) override fun truncateText(text: String, maxTokens: Int): String = encoding.truncateText(text, maxTokens) @@ -41,9 +40,9 @@ class OpenAIFunChat( fun Encoding.countTokensFromMessages(tokensPerMessage: Int, tokensPerName: Int): Int = messages.sumOf { message -> countTokens(message.role.name) + - countTokens(message.content) + - tokensPerMessage + - tokensPerName + countTokens(message.content) + + tokensPerMessage + + tokensPerName } + 3 return encoding.countTokensFromMessages(tokensPerMessage = 5, tokensPerName = 5) + 10 } diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt index 316f70122..885a5bcc5 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIModel.kt @@ -1,15 +1,11 @@ package com.xebia.functional.xef.conversation.llm.openai.models -import com.xebia.functional.tokenizer.Encoding import com.xebia.functional.tokenizer.EncodingType -import com.xebia.functional.tokenizer.truncateText import com.xebia.functional.xef.llm.LLM -import com.xebia.functional.xef.llm.models.chat.Message interface OpenAIModel : LLM { val encodingType: EncodingType val encoding get() = encodingType.encoding - } From 1f19f61ee3f3be348b085e91afabca4e8012db7d Mon Sep 17 00:00:00 2001 From: ron Date: Fri, 24 Nov 2023 23:12:15 +0100 Subject: [PATCH 16/19] removes some intermediary functions in LLM and adapts all usages --- .../com/xebia/functional/xef/llm/BaseChat.kt | 10 ++++----- .../xebia/functional/xef/llm/Completion.kt | 4 ++-- .../com/xebia/functional/xef/llm/LLM.kt | 22 ------------------- .../functional/xef/llm/PromptCalculator.kt | 4 ++-- .../xef/store/CombinedVectorStore.kt | 4 ++-- .../functional/xef/store/LocalVectorStore.kt | 4 ++-- .../xebia/functional/xef/store/MemoryUtils.kt | 4 ++-- .../xebia/functional/xef/store/VectorStore.kt | 10 ++++++--- .../xef/conversation/ConversationSpec.kt | 12 +++++----- .../functional/xef/data/TestFunctionsModel.kt | 9 +++++--- .../xebia/functional/xef/data/TestModel.kt | 9 +++++--- .../com/xebia/functional/xef/store/Lucene.kt | 3 ++- .../xef/store/PostgreSQLVectorStore.kt | 3 ++- .../src/test/kotlin/xef/PGVectorStoreSpec.kt | 4 ++++ .../llm/openai/models/OpenAIImages.kt | 5 ----- 15 files changed, 48 insertions(+), 59 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt index 216339314..9963caf63 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt @@ -10,16 +10,16 @@ interface BaseChat : LLM { @Deprecated( "will be removed from LLM in favor of abstracting former ModelType, use contextLength instead" ) - override val maxContextLength + val maxContextLength get() = (contextLength as? MaxIoContextLength.Combined)?.total ?: error( - "accessing maxContextLength requires model's context length to be of type ${MaxIoContextLength.Combined::class.qualifiedName}" + "accessing maxContextLength requires model's context length to be of type MaxIoContextLength.Combined" ) - @Suppress("OVERRIDE_DEPRECATION") override fun countTokens(text: String): Int + @Suppress("OVERRIDE_DEPRECATION") fun countTokens(text: String): Int - @Suppress("OVERRIDE_DEPRECATION") override fun truncateText(text: String, maxTokens: Int): String + @Suppress("OVERRIDE_DEPRECATION") fun truncateText(text: String, maxTokens: Int): String - @Suppress("OVERRIDE_DEPRECATION") override fun tokensFromMessages(messages: List): Int + @Suppress("OVERRIDE_DEPRECATION") fun tokensFromMessages(messages: List): Int } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt index 898679872..41ca3d481 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/Completion.kt @@ -9,7 +9,7 @@ interface Completion : LLM { suspend fun createCompletion(request: CompletionRequest): CompletionResult - override fun countTokens(text: String): Int + fun countTokens(text: String): Int - override fun truncateText(text: String, maxTokens: Int): String + fun truncateText(text: String, maxTokens: Int): String } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt index 8ef60ec4d..d94dc63ca 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/LLM.kt @@ -1,7 +1,6 @@ package com.xebia.functional.xef.llm import com.xebia.functional.xef.llm.models.ModelID -import com.xebia.functional.xef.llm.models.chat.Message // sealed modifier temporarily removed as OAI's implementation of tokensFromMessages has to extend // and override LLM @@ -19,26 +18,5 @@ import com.xebia.functional.xef.llm.models.chat.Message */ fun copy(modelID: ModelID): LLM - @Deprecated( - "will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" - ) - fun tokensFromMessages(messages: List): Int = TODO() // intermediary - - @Deprecated( - "will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" - ) - fun countTokens(text: String): Int = TODO() // intermediary - - @Deprecated( - "will be moved out of LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" - ) - fun truncateText(text: String, maxTokens: Int): String = TODO() // intermediary - - @Deprecated( - "will be removed from LLM in favor of abstracting former ModelType, as this is not inherent to all LLMs" - ) - val maxContextLength: Int - get() = TODO() - override fun close() = Unit } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt index 3dc27fcad..3edc99044 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt @@ -107,7 +107,7 @@ internal object PromptCalculator { return maxHistoryTokens } - private fun calculateRemainingTokensForContext(llm: LLM, prompt: Prompt): Int { + private fun calculateRemainingTokensForContext(llm: BaseChat, prompt: Prompt): Int { val maxContextLength: Int = llm.maxContextLength val remainingTokens: Int = maxContextLength - prompt.configuration.minResponseTokens @@ -121,6 +121,6 @@ internal object PromptCalculator { return remainingTokensForContexts } - private suspend fun Conversation.memories(llm: LLM, limitTokens: Int): List = + private suspend fun Conversation.memories(llm: BaseChat, limitTokens: Int): List = conversationId?.let { store.memories(llm, it, limitTokens) } ?: emptyList() } diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/CombinedVectorStore.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/CombinedVectorStore.kt index eb08e79e5..eccc9ee0c 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/CombinedVectorStore.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/CombinedVectorStore.kt @@ -1,6 +1,6 @@ package com.xebia.functional.xef.store -import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.BaseChat import com.xebia.functional.xef.llm.models.embeddings.Embedding /** @@ -13,7 +13,7 @@ class CombinedVectorStore(private val top: VectorStore, private val bottom: Vect VectorStore by top { override suspend fun memories( - llm: LLM, + llm: BaseChat, conversationId: ConversationId, limitTokens: Int ): List { diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/LocalVectorStore.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/LocalVectorStore.kt index e14720ca8..a5458eb47 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/LocalVectorStore.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/LocalVectorStore.kt @@ -4,8 +4,8 @@ import arrow.atomic.Atomic import arrow.atomic.AtomicInt import arrow.atomic.getAndUpdate import arrow.atomic.update +import com.xebia.functional.xef.llm.BaseChat import com.xebia.functional.xef.llm.Embeddings -import com.xebia.functional.xef.llm.LLM import com.xebia.functional.xef.llm.models.embeddings.Embedding import com.xebia.functional.xef.llm.models.embeddings.RequestConfig import kotlin.math.sqrt @@ -55,7 +55,7 @@ private constructor(private val embeddings: Embeddings, private val state: Atomi } override suspend fun memories( - llm: LLM, + llm: BaseChat, conversationId: ConversationId, limitTokens: Int ): List { diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/MemoryUtils.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/MemoryUtils.kt index 392719f58..82368033d 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/MemoryUtils.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/MemoryUtils.kt @@ -1,8 +1,8 @@ package com.xebia.functional.xef.store -import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.BaseChat -fun List.reduceByLimitToken(llm: LLM, limitTokens: Int): List { +fun List.reduceByLimitToken(llm: BaseChat, limitTokens: Int): List { val tokensFromMessages = llm.tokensFromMessages(map { it.content }) return if (tokensFromMessages <= limitTokens) this else diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/VectorStore.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/VectorStore.kt index 031825ed0..17e4ee307 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/store/VectorStore.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/store/VectorStore.kt @@ -1,7 +1,7 @@ package com.xebia.functional.xef.store import arrow.atomic.AtomicInt -import com.xebia.functional.xef.llm.LLM +import com.xebia.functional.xef.llm.BaseChat import com.xebia.functional.xef.llm.models.embeddings.Embedding import kotlin.jvm.JvmStatic @@ -15,7 +15,11 @@ interface VectorStore { suspend fun addMemories(memories: List) - suspend fun memories(llm: LLM, conversationId: ConversationId, limitTokens: Int): List + suspend fun memories( + llm: BaseChat, + conversationId: ConversationId, + limitTokens: Int + ): List /** * Add texts to the vector store after running them through the embeddings @@ -56,7 +60,7 @@ interface VectorStore { override suspend fun addMemories(memories: List) {} override suspend fun memories( - llm: LLM, + llm: BaseChat, conversationId: ConversationId, limitTokens: Int ): List = emptyList() diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index 548719acb..557b1fdc5 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -274,7 +274,7 @@ class ConversationSpec : "when using MessagesToHistory.ALL policy, the scope's store should contains all messages" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) @@ -302,7 +302,7 @@ class ConversationSpec : "when using MessagesToHistory.ONLY_SYSTEM_MESSAGES policy, the scope's store should contains only system messages" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) @@ -330,7 +330,7 @@ class ConversationSpec : "when using MessagesToHistory.NOT_SYSTEM_MESSAGES policy, the scope's store shouldn't contains system messages" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) @@ -358,7 +358,7 @@ class ConversationSpec : "when using MessagesToHistory.NONE policy, the scope's store shouldn't contains messages" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) @@ -389,7 +389,7 @@ class ConversationSpec : "when using MessagesFromHistory.ALL policy, the request should contains the previous messages in the conversation" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) @@ -423,7 +423,7 @@ class ConversationSpec : "when using MessagesFromHistory.NONE policy, the request shouldn't contains the previous messages in the conversation" { val conversationId = ConversationId(UUID.generateUUID().toString()) - val model = TestModel(modelType = ModelType.ADA) + val model = TestModel() val vectorStore = LocalVectorStore(TestEmbeddings()) diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt index cc29e535b..036f9b76e 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestFunctionsModel.kt @@ -23,9 +23,12 @@ class TestFunctionsModel( override fun copy(modelID: ModelID) = TestFunctionsModel(contextLength, responses) - override fun tokensFromMessages(messages: List): Int { - return messages.sumOf { it.content.length } - } + override fun countTokens(text: String): Int = text.length + + override fun truncateText(text: String, maxTokens: Int): String = text + + override fun tokensFromMessages(messages: List): Int = + messages.sumOf { it.content.length } override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { return EmbeddingResult(data = emptyList(), usage = Usage.ZERO) diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt index 8f5d4ad87..f75953ba0 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/data/TestModel.kt @@ -53,9 +53,12 @@ class TestModel( throw NotImplementedError() } - override fun tokensFromMessages(messages: List): Int { - return messages.sumOf { it.content.length } - } + override fun countTokens(text: String): Int = text.length + + override fun truncateText(text: String, maxTokens: Int): String = text + + override fun tokensFromMessages(messages: List): Int = + messages.sumOf { it.content.length } override suspend fun createEmbeddings(request: EmbeddingRequest): EmbeddingResult { return EmbeddingResult(data = emptyList(), usage = Usage.ZERO) diff --git a/integrations/lucene/src/main/kotlin/com/xebia/functional/xef/store/Lucene.kt b/integrations/lucene/src/main/kotlin/com/xebia/functional/xef/store/Lucene.kt index 27a97d590..43a2e9c51 100644 --- a/integrations/lucene/src/main/kotlin/com/xebia/functional/xef/store/Lucene.kt +++ b/integrations/lucene/src/main/kotlin/com/xebia/functional/xef/store/Lucene.kt @@ -1,6 +1,7 @@ package com.xebia.functional.xef.store import arrow.atomic.AtomicInt +import com.xebia.functional.xef.llm.BaseChat import com.xebia.functional.xef.llm.Embeddings import com.xebia.functional.xef.llm.LLM import com.xebia.functional.xef.llm.models.chat.Message @@ -52,7 +53,7 @@ open class Lucene( writer.commit() } - override suspend fun memories(llm: LLM, conversationId: ConversationId, limitTokens: Int): List = + override suspend fun memories(llm: BaseChat, conversationId: ConversationId, limitTokens: Int): List = getMemoryByConversationId(conversationId).reduceByLimitToken(llm, limitTokens).reversed() override suspend fun addTexts(texts: List) { diff --git a/integrations/postgresql/src/main/kotlin/com/xebia/functional/xef/store/PostgreSQLVectorStore.kt b/integrations/postgresql/src/main/kotlin/com/xebia/functional/xef/store/PostgreSQLVectorStore.kt index 3ffcbc66d..bc0754381 100644 --- a/integrations/postgresql/src/main/kotlin/com/xebia/functional/xef/store/PostgreSQLVectorStore.kt +++ b/integrations/postgresql/src/main/kotlin/com/xebia/functional/xef/store/PostgreSQLVectorStore.kt @@ -1,6 +1,7 @@ package com.xebia.functional.xef.store import arrow.atomic.AtomicInt +import com.xebia.functional.xef.llm.BaseChat import com.xebia.functional.xef.llm.Embeddings import com.xebia.functional.xef.llm.LLM import com.xebia.functional.xef.llm.models.chat.Message @@ -43,7 +44,7 @@ class PGVectorStore( } } - override suspend fun memories(llm: LLM, conversationId: ConversationId, limitTokens: Int): List = + override suspend fun memories(llm: BaseChat, conversationId: ConversationId, limitTokens: Int): List = getMemoryByConversationId(conversationId).reduceByLimitToken(llm, limitTokens).reversed() private fun JDBCSyntax.getCollection(collectionName: String): PGCollection = diff --git a/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt b/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt index 4f0a722fd..279d5529d 100644 --- a/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt +++ b/integrations/postgresql/src/test/kotlin/xef/PGVectorStoreSpec.kt @@ -109,6 +109,10 @@ class TestLLM : Chat, AutoCloseable { override fun tokensFromMessages(messages: List): Int = messages.map { calculateTokens(it) }.sum() + override fun countTokens(text: String): Int = text.length + + override fun truncateText(text: String, maxTokens: Int): String = text + private fun calculateTokens(message: Message): Int = message.content.split(" ").size + 2 // 2 is the role and name override suspend fun createChatCompletion(request: ChatCompletionRequest): ChatCompletionResponse { diff --git a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt index 3e17afc33..74fb6d789 100644 --- a/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt +++ b/openai/src/commonMain/kotlin/com/xebia/functional/xef/conversation/llm/openai/models/OpenAIImages.kt @@ -8,7 +8,6 @@ import com.xebia.functional.tokenizer.EncodingType import com.xebia.functional.xef.conversation.llm.openai.OpenAI import com.xebia.functional.xef.llm.Images import com.xebia.functional.xef.llm.models.ModelID -import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.llm.models.images.ImageGenerationUrl import com.xebia.functional.xef.llm.models.images.ImagesGenerationRequest import com.xebia.functional.xef.llm.models.images.ImagesGenerationResponse @@ -34,8 +33,4 @@ class OpenAIImages( val response = client.imageURL(clientRequest) return ImagesGenerationResponse(data = response.map { ImageGenerationUrl(it.url) }) } - - override fun tokensFromMessages(messages: List): Int { - TODO("Not yet implemented") - } } From 253adbaf86fda476d74f0945787c81e741bfcfe8 Mon Sep 17 00:00:00 2001 From: ron Date: Sat, 25 Nov 2023 21:55:04 +0100 Subject: [PATCH 17/19] fix potential bug with integer overflow in PromptCalculator --- .../kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt index 3edc99044..c76a63ac3 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/PromptCalculator.kt @@ -8,6 +8,8 @@ import com.xebia.functional.xef.llm.models.chat.Message import com.xebia.functional.xef.prompt.Prompt import com.xebia.functional.xef.prompt.templates.assistant import com.xebia.functional.xef.store.Memory +import kotlin.math.floor +import kotlin.math.roundToInt internal object PromptCalculator { @@ -97,13 +99,13 @@ internal object PromptCalculator { private fun calculateMaxContextTokens(prompt: Prompt, remainingTokensForContexts: Int): Int { val contextPercent = prompt.configuration.messagePolicy.contextPercent - val maxContextTokens = (remainingTokensForContexts * contextPercent) / 100 + val maxContextTokens = floor(remainingTokensForContexts * (contextPercent / 100f)).roundToInt() return maxContextTokens } private fun calculateMaxHistoryTokens(prompt: Prompt, remainingTokensForContexts: Int): Int { val historyPercent = prompt.configuration.messagePolicy.historyPercent - val maxHistoryTokens = (remainingTokensForContexts * historyPercent) / 100 + val maxHistoryTokens = floor(remainingTokensForContexts * (historyPercent / 100f)).roundToInt() return maxHistoryTokens } From 066135e21ae7f29d6467d5a902fef78c1bee6357 Mon Sep 17 00:00:00 2001 From: ron Date: Sat, 25 Nov 2023 22:05:08 +0100 Subject: [PATCH 18/19] fixes test (wrong model contxt length) --- .../kotlin/com/xebia/functional/xef/llm/BaseChat.kt | 6 +++--- .../xebia/functional/xef/conversation/ConversationSpec.kt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt index 9963caf63..1933ecdb6 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/llm/BaseChat.kt @@ -17,9 +17,9 @@ interface BaseChat : LLM { "accessing maxContextLength requires model's context length to be of type MaxIoContextLength.Combined" ) - @Suppress("OVERRIDE_DEPRECATION") fun countTokens(text: String): Int + fun countTokens(text: String): Int - @Suppress("OVERRIDE_DEPRECATION") fun truncateText(text: String, maxTokens: Int): String + fun truncateText(text: String, maxTokens: Int): String - @Suppress("OVERRIDE_DEPRECATION") fun tokensFromMessages(messages: List): Int + fun tokensFromMessages(messages: List): Int } diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index 557b1fdc5..0b5b6129e 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -63,7 +63,7 @@ class ConversationSpec : ) val vectorStore = scope.store - val modelAda = TestModel(responses = messages) + val modelAda = TestModel(responses = messages, contextLength = MaxIoContextLength.Combined(2049)) val totalTokens = modelAda.tokensFromMessages( From 295ed8cc382f2368cc7d20e950646238973834be Mon Sep 17 00:00:00 2001 From: Intex32 Date: Sat, 25 Nov 2023 21:06:32 +0000 Subject: [PATCH 19/19] Apply spotless formatting --- .../com/xebia/functional/xef/conversation/ConversationSpec.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt index 0b5b6129e..2d2d0b6ac 100644 --- a/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt +++ b/core/src/commonTest/kotlin/com/xebia/functional/xef/conversation/ConversationSpec.kt @@ -63,7 +63,8 @@ class ConversationSpec : ) val vectorStore = scope.store - val modelAda = TestModel(responses = messages, contextLength = MaxIoContextLength.Combined(2049)) + val modelAda = + TestModel(responses = messages, contextLength = MaxIoContextLength.Combined(2049)) val totalTokens = modelAda.tokensFromMessages(