From 63de352f7a678a31c99b4beb961378e00a9b5437 Mon Sep 17 00:00:00 2001 From: raulraja Date: Mon, 27 May 2024 16:47:04 +0200 Subject: [PATCH 01/12] Enum/Classification support for models that do not support `logitBias` --- .../kotlin/com/xebia/functional/xef/AI.kt | 15 +++- .../kotlin/com/xebia/functional/xef/Config.kt | 3 +- .../com/xebia/functional/xef/DefaultAI.kt | 74 +++++++++++++++++-- .../configuration/PromptConfiguration.kt | 1 + .../functional/xef/dsl/chat/EnumOllama.kt | 18 +++++ 5 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/AI.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/AI.kt index 81276970b..81dd00a68 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/AI.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/AI.kt @@ -6,6 +6,7 @@ import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestMo import com.xebia.functional.xef.conversation.AiDsl import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.prompt.Prompt +import com.xebia.functional.xef.prompt.configuration.PromptConfiguration import kotlin.coroutines.cancellation.CancellationException import kotlin.reflect.KClass import kotlin.reflect.KType @@ -111,7 +112,19 @@ sealed interface AI { config: Config = Config(), api: Chat = OpenAI(config).chat, conversation: Conversation = Conversation() - ): A = chat(Prompt(model, prompt), target, config, api, conversation) + ): A = + chat( + prompt = + Prompt( + model = model, + value = prompt, + configuration = PromptConfiguration { supportsLogitBias = config.supportsLogitBias } + ), + target = target, + config = config, + api = api, + conversation = conversation + ) @AiDsl suspend inline operator fun invoke( diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt index 4645fbd4f..189ceba0c 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt @@ -26,7 +26,8 @@ data class Config( classDiscriminator = "_type_" }, val streamingPrefix: String = "data:", - val streamingDelimiter: String = "data: [DONE]" + val streamingDelimiter: String = "data: [DONE]", + val supportsLogitBias: Boolean = true, ) { companion object { val DEFAULT = Config() diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt index 4d3b1b9cd..b06d8ca88 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt @@ -9,6 +9,8 @@ import com.xebia.functional.xef.llm.models.modelType import com.xebia.functional.xef.llm.prompt import com.xebia.functional.xef.llm.promptStreaming import com.xebia.functional.xef.prompt.Prompt +import com.xebia.functional.xef.prompt.PromptBuilder.Companion.user +import com.xebia.functional.xef.prompt.contentAsString import kotlin.reflect.KClass import kotlin.reflect.KType import kotlin.reflect.typeOf @@ -50,7 +52,7 @@ data class DefaultAI( val serializer = serializer() return when (serializer.descriptor.kind) { SerialKind.ENUM -> { - runWithEnumSingleTokenSerializer(serializer, prompt) + runWithEnumSerializer(serializer, prompt) } // else -> runWithSerializer(prompt, serializer) PolymorphicKind.OPEN -> @@ -82,11 +84,61 @@ data class DefaultAI( } } - @OptIn(ExperimentalSerializationApi::class) - suspend fun runWithEnumSingleTokenSerializer(serializer: KSerializer, prompt: Prompt): A { + private suspend fun runWithEnumSerializer(serializer: KSerializer, prompt: Prompt): A = + if (prompt.configuration.supportsLogitBias) { + runWithEnumSingleTokenSerializer(serializer, prompt) + } else { + runWithEnumRegexResponseSerializer(serializer, prompt) + } + + private suspend fun runWithEnumRegexResponseSerializer( + serializer: KSerializer, + prompt: Prompt + ): A { + val cases = casesFromEnumSerializer(serializer) + val classificationMessage = + user( + """ + + You are an AI, expected to classify the `context` into one of the `cases`: + + ${prompt.messages.joinToString("\n") { it.contentAsString() }} + + ${cases.mapIndexed { index, s -> "$s" }.joinToString("\n")} + + Select the `case` corresponding to the `context`. + IMPORTANT. Reply exclusively with the selected `case`. + + """ + .trimIndent() + ) + val result = + api.createChatCompletion( + CreateChatCompletionRequest( + messages = prompt.messages + classificationMessage, + model = model, + maxTokens = prompt.configuration.maxTokens, + temperature = 0.0 + ) + ) + val casesRegexes = cases.map { ".*$it.*" } + val responseContent = result.choices[0].message.content ?: "" + val choice = + casesRegexes + .zip(cases) + .firstOrNull { + Regex(it.first, RegexOption.IGNORE_CASE).containsMatchIn(responseContent.trim()) + } + ?.second + return serializeWithEnumSerializer(choice, enumSerializer) + } + + private suspend fun runWithEnumSingleTokenSerializer( + serializer: KSerializer, + prompt: Prompt + ): A { val encoding = model.modelType(forFunctions = false).encoding - val cases = - serializer.descriptor.elementDescriptors.map { it.serialName.substringAfterLast(".") } + val cases = casesFromEnumSerializer(serializer) val logitBias = cases .flatMap { @@ -108,7 +160,17 @@ data class DefaultAI( ) ) val choice = result.choices[0].message.content - val enumSerializer = enumSerializer + return serializeWithEnumSerializer(choice, enumSerializer) + } + + @OptIn(ExperimentalSerializationApi::class) + private fun casesFromEnumSerializer(serializer: KSerializer): List = + serializer.descriptor.elementDescriptors.map { it.serialName.substringAfterLast(".") } + + private fun serializeWithEnumSerializer( + choice: String?, + enumSerializer: ((case: String) -> A)? + ): A { return if (choice != null && enumSerializer != null) { enumSerializer(choice) } else { diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/prompt/configuration/PromptConfiguration.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/prompt/configuration/PromptConfiguration.kt index b8d42db2b..9ca770661 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/prompt/configuration/PromptConfiguration.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/prompt/configuration/PromptConfiguration.kt @@ -18,6 +18,7 @@ constructor( var maxTokens: Int = 500, var messagePolicy: MessagePolicy = MessagePolicy(), var seed: Int? = null, + var supportsLogitBias: Boolean = true, ) { fun messagePolicy(block: MessagePolicy.() -> Unit) = messagePolicy.apply { block() } diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt b/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt new file mode 100644 index 000000000..eee8e2c1d --- /dev/null +++ b/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt @@ -0,0 +1,18 @@ +package com.xebia.functional.xef.dsl.chat + +import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel +import com.xebia.functional.xef.AI +import com.xebia.functional.xef.Config +import com.xebia.functional.xef.OpenAI + +suspend fun main() { + val config = Config(baseUrl = "http://localhost:11434/v1/", supportsLogitBias = false) + val sentiment = + AI( + prompt = "I love Xef!", + model = CreateChatCompletionRequestModel.Custom("gemma:2b"), + config = config, + api = OpenAI(config, logRequests = true).chat, + ) + println(sentiment) // positive +} From 3f28a1ec2800843fce71f7e9ee16b3414ddc1c83 Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 08:09:12 +0200 Subject: [PATCH 02/12] Add testcontainers ollama and classification test for gemma 2b --- core/build.gradle.kts | 5 +- .../ollama/tests/EnumClassificationTest.kt | 34 +++++++ .../xef/ollama/tests/OllamaTests.kt | 93 +++++++++++++++++++ .../xef/ollama/tests/models/OllamaModels.kt | 8 ++ .../xef/ollama/tests/models/Sentiment.kt | 12 +++ core/src/jvmTest/resources/logback.xml | 26 ++++++ .../functional/xef/dsl/chat/EnumOllama.kt | 2 +- examples/src/main/resources/logback.xml | 2 +- gradle/libs.versions.toml | 3 +- 9 files changed, 181 insertions(+), 4 deletions(-) create mode 100644 core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt create mode 100644 core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt create mode 100644 core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/OllamaModels.kt create mode 100644 core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt create mode 100644 core/src/jvmTest/resources/logback.xml diff --git a/core/build.gradle.kts b/core/build.gradle.kts index de1530459..bc3d922f3 100644 --- a/core/build.gradle.kts +++ b/core/build.gradle.kts @@ -106,7 +106,10 @@ kotlin { } val jvmTest by getting { dependencies { - implementation(libs.kotest.junit5) + implementation(libs.ollama.testcontainers) + implementation(libs.junit.jupiter.api) + implementation(libs.junit.jupiter.engine) + implementation(libs.logback) } } val linuxX64Main by getting { diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt new file mode 100644 index 000000000..2d9553cc8 --- /dev/null +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt @@ -0,0 +1,34 @@ +package com.xebia.functional.xef.ollama.tests + +import com.xebia.functional.xef.ollama.tests.models.OllamaModels +import com.xebia.functional.xef.ollama.tests.models.Sentiment +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test + +class EnumClassificationTest : OllamaTests() { + @Test + fun `enum classification`() { + runBlocking { + val models = setOf(OllamaModels.Gemma2B) + val sentiments = + ollama( + models = models, + prompt = "The sentiment of this text is positive.", + ) + expectSentiment(Sentiment.POSITIVE, sentiments, models) + } + } + + private fun expectSentiment( + expected: Sentiment, + sentiments: List, + models: Set + ) { + assert(sentiments.size == models.size) { + "Expected ${models.size} results but got ${sentiments.size}" + } + sentiments.forEach { sentiment -> + assert(sentiment == expected) { "Expected $expected but got $sentiment" } + } + } +} diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt new file mode 100644 index 000000000..8dd7d4697 --- /dev/null +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt @@ -0,0 +1,93 @@ +package com.xebia.functional.xef.ollama.tests + +import arrow.fx.coroutines.parMap +import com.xebia.functional.openai.generated.api.Chat +import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel +import com.xebia.functional.xef.AI +import com.xebia.functional.xef.Config +import com.xebia.functional.xef.OpenAI +import io.github.oshai.kotlinlogging.KotlinLogging +import kotlinx.coroutines.Dispatchers +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.BeforeAll +import org.testcontainers.ollama.OllamaContainer +import org.testcontainers.utility.DockerImageName + +abstract class OllamaTests { + + val logger = KotlinLogging.logger {} + + companion object { + private const val OLLAMA_IMAGE = "ollama/ollama:0.1.26" + private const val NEW_IMAGE_NAME = "ollama/ollama:test" + + val ollama: OllamaContainer by lazy { + // check if the new image is already present otherwise pull the image + if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) == null) { + OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) + } else { + OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) + } + } + + @BeforeAll + @JvmStatic + fun setup() { + ollama.start() + ollama.commitToImage(NEW_IMAGE_NAME) + } + + @AfterAll + @JvmStatic + fun teardown() { + ollama.commitToImage(NEW_IMAGE_NAME) + ollama.stop() + } + } + + suspend inline fun ollama( + models: Set, + prompt: String, + config: Config = Config(baseUrl = ollamaBaseUrl(), supportsLogitBias = false), + api: Chat = OpenAI(config = config, logRequests = true).chat, + ): List { + // pull all models + models.parMap(context = Dispatchers.IO) { model -> + logger.info { "🚒 Pulling model $model" } + val pullResult = ollama.execInContainer("ollama", "pull", model) + if (pullResult.exitCode != 0) { + logger.error { pullResult.stderr } + throw RuntimeException("Failed to pull model $model") + } + logger.info { pullResult.stdout } + logger.info { "🚒 Pulled $model" } + } + // run all models + models.parMap(context = Dispatchers.IO) { model -> + logger.info { "πŸš€ Starting model $model" } + val runResult = ollama.execInContainer("ollama", "run", model) + if (runResult.exitCode != 0) { + logger.error { runResult.stderr } + throw RuntimeException("Failed to run model $model") + } + logger.info { runResult.stdout } + println("πŸš€ Started $model") + } + // run inference on all models + return models.parMap(context = Dispatchers.IO) { model -> + logger.info { "πŸš€ Running inference on model $model" } + val result: A = + AI( + prompt = prompt, + config = config, + api = api, + model = CreateChatCompletionRequestModel.Custom(model), + ) + logger.info { "πŸš€ Inference on model $model: $result" } + result + } + } + + fun ollamaBaseUrl(): String = + "http://${ollama.host}:${ollama.getMappedPort(ollama.exposedPorts.first())}/v1/" +} diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/OllamaModels.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/OllamaModels.kt new file mode 100644 index 000000000..76b5a8fca --- /dev/null +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/OllamaModels.kt @@ -0,0 +1,8 @@ +package com.xebia.functional.xef.ollama.tests.models + +object OllamaModels { + const val Gemma2B = "gemma:2b" + const val Phi3Latest = "phi3:latest" + const val LLama3_8B = "llama3:8b" + const val Qwen0_5B = "qwen:0.5b" +} diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt new file mode 100644 index 000000000..b83d008c4 --- /dev/null +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt @@ -0,0 +1,12 @@ +package com.xebia.functional.xef.ollama.tests.models + +import kotlinx.serialization.Serializable + +@Serializable +enum class Sentiment { + POSITIVE, + NEGATIVE, + NEUTRAL, + MIXED, + UNKNOWN +} diff --git a/core/src/jvmTest/resources/logback.xml b/core/src/jvmTest/resources/logback.xml new file mode 100644 index 000000000..fb058d664 --- /dev/null +++ b/core/src/jvmTest/resources/logback.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} -%kvp- %msg%n + + + + + + + + + + + + + + + diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt b/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt index eee8e2c1d..eaf647c06 100644 --- a/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt +++ b/examples/src/main/kotlin/com/xebia/functional/xef/dsl/chat/EnumOllama.kt @@ -10,7 +10,7 @@ suspend fun main() { val sentiment = AI( prompt = "I love Xef!", - model = CreateChatCompletionRequestModel.Custom("gemma:2b"), + model = CreateChatCompletionRequestModel.Custom("orca-mini:3b"), config = config, api = OpenAI(config, logRequests = true).chat, ) diff --git a/examples/src/main/resources/logback.xml b/examples/src/main/resources/logback.xml index fb058d664..9a90533b5 100644 --- a/examples/src/main/resources/logback.xml +++ b/examples/src/main/resources/logback.xml @@ -12,7 +12,7 @@ - + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e064e4c66..316b10e88 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -15,7 +15,7 @@ kotest-arrow = "1.4.0" klogging = "6.0.9" uuid = "0.0.22" postgresql = "42.7.3" -testcontainers = "1.19.5" +testcontainers = "1.19.7" hikari = "5.1.0" dokka = "1.9.20" logback = "1.5.5" @@ -115,6 +115,7 @@ opentelemetry-extension-kotlin = { module = "io.opentelemetry:opentelemetry-exte progressbar = { module = "me.tongfei:progressbar", version.ref = "progressbar" } jmf = { module = "javax.media:jmf", version.ref = "jmf" } mp3-wav-converter = { module = "com.sipgate:mp3-wav", version.ref = "mp3-wav-converter" } +ollama-testcontainers = { module = "org.testcontainers:ollama", version.ref = "testcontainers" } From d925081be26e463698369f4c2f2e960202f3b7c5 Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 08:16:52 +0200 Subject: [PATCH 03/12] attempt to fix container load on CI --- .../com/xebia/functional/xef/ollama/tests/OllamaTests.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt index 8dd7d4697..581098282 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt @@ -23,10 +23,10 @@ abstract class OllamaTests { val ollama: OllamaContainer by lazy { // check if the new image is already present otherwise pull the image - if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) == null) { - OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) - } else { + if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) != null) { OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) + } else { + OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) } } From 71c7c7c739af8d7a1f01d765ef774d0421aeaee6 Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 08:23:13 +0200 Subject: [PATCH 04/12] attempt to fix container load on CI. 2 --- .../functional/xef/ollama/tests/OllamaTests.kt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt index 581098282..0b0488346 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt @@ -19,28 +19,30 @@ abstract class OllamaTests { companion object { private const val OLLAMA_IMAGE = "ollama/ollama:0.1.26" - private const val NEW_IMAGE_NAME = "ollama/ollama:test" + // private const val NEW_IMAGE_NAME = "ollama/ollama:test" val ollama: OllamaContainer by lazy { // check if the new image is already present otherwise pull the image - if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) != null) { - OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) - } else { - OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) - } + // if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) != + // null) { + // OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) + // } else { + // OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) + // } + OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) } @BeforeAll @JvmStatic fun setup() { ollama.start() - ollama.commitToImage(NEW_IMAGE_NAME) + // ollama.commitToImage(NEW_IMAGE_NAME) } @AfterAll @JvmStatic fun teardown() { - ollama.commitToImage(NEW_IMAGE_NAME) + // ollama.commitToImage(NEW_IMAGE_NAME) ollama.stop() } } From aba9d4f816abcfd004cbe1d9264dd65017badd42 Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 08:29:04 +0200 Subject: [PATCH 05/12] do not bail if there is no token --- core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt index 189ceba0c..7d9b678ae 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt @@ -1,6 +1,5 @@ package com.xebia.functional.xef -import arrow.core.nonEmptyListOf import com.xebia.functional.openai.Config as OpenAIConfig import com.xebia.functional.openai.generated.api.OpenAI import com.xebia.functional.xef.env.getenv @@ -51,7 +50,8 @@ fun OpenAI( val token = config.token ?: getenv(KEY_ENV_VAR) - ?: throw AIError.Env.OpenAI(nonEmptyListOf("missing $KEY_ENV_VAR env var")) + ?: "" // throw AIError.Env.OpenAI(nonEmptyListOf("missing $KEY_ENV_VAR env + // var")) val clientConfig: HttpClientConfig<*>.() -> Unit = { install(ContentNegotiation) { json(config.json) } install(HttpTimeout) { From 766feff7a481dc7f53a10121783e80002b666887 Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 08:57:10 +0200 Subject: [PATCH 06/12] try with llama3 8b --- .../src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt | 2 +- .../xebia/functional/xef/ollama/tests/EnumClassificationTest.kt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt index b06d8ca88..5b65d0c39 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/DefaultAI.kt @@ -104,7 +104,7 @@ data class DefaultAI( ${prompt.messages.joinToString("\n") { it.contentAsString() }} - ${cases.mapIndexed { index, s -> "$s" }.joinToString("\n")} + ${cases.map { s -> "$s" }.joinToString("\n")} Select the `case` corresponding to the `context`. IMPORTANT. Reply exclusively with the selected `case`. diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt index 2d9553cc8..9f3e7796b 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt @@ -9,7 +9,7 @@ class EnumClassificationTest : OllamaTests() { @Test fun `enum classification`() { runBlocking { - val models = setOf(OllamaModels.Gemma2B) + val models = setOf(OllamaModels.LLama3_8B) val sentiments = ollama( models = models, From 05a9189b730eaee03f40b9a55bc551d512ca409c Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 09:05:56 +0200 Subject: [PATCH 07/12] try with llama3 8b (separate test cases) --- .../ollama/tests/EnumClassificationTest.kt | 22 ++++++++++--------- .../ollama/tests/EnumClassificationTest2.kt | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt index 9f3e7796b..7a940ebcc 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt @@ -19,16 +19,18 @@ class EnumClassificationTest : OllamaTests() { } } - private fun expectSentiment( - expected: Sentiment, - sentiments: List, - models: Set - ) { - assert(sentiments.size == models.size) { - "Expected ${models.size} results but got ${sentiments.size}" - } - sentiments.forEach { sentiment -> - assert(sentiment == expected) { "Expected $expected but got $sentiment" } + companion object { + internal fun expectSentiment( + expected: Sentiment, + sentiments: List, + models: Set + ) { + assert(sentiments.size == models.size) { + "Expected ${models.size} results but got ${sentiments.size}" + } + sentiments.forEach { sentiment -> + assert(sentiment == expected) { "Expected $expected but got $sentiment" } + } } } } diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt new file mode 100644 index 000000000..8cf08deb0 --- /dev/null +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt @@ -0,0 +1,22 @@ +package com.xebia.functional.xef.ollama.tests + +import com.xebia.functional.xef.ollama.tests.EnumClassificationTest.Companion.expectSentiment +import com.xebia.functional.xef.ollama.tests.models.OllamaModels +import com.xebia.functional.xef.ollama.tests.models.Sentiment +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test + +class EnumClassificationTest2 : OllamaTests() { + @Test + fun `enum classification 2`() { + runBlocking { + val models = setOf(OllamaModels.LLama3_8B) + val sentiments = + ollama( + models = models, + prompt = "The sentiment of this text is negative.", + ) + expectSentiment(Sentiment.NEGATIVE, sentiments, models) + } + } +} From 6cbf75469998edcd434ea7cf4e97a08335f551fd Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 09:15:02 +0200 Subject: [PATCH 08/12] try saving docker image --- .../functional/xef/ollama/tests/OllamaTests.kt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt index 0b0488346..581098282 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt @@ -19,30 +19,28 @@ abstract class OllamaTests { companion object { private const val OLLAMA_IMAGE = "ollama/ollama:0.1.26" - // private const val NEW_IMAGE_NAME = "ollama/ollama:test" + private const val NEW_IMAGE_NAME = "ollama/ollama:test" val ollama: OllamaContainer by lazy { // check if the new image is already present otherwise pull the image - // if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) != - // null) { - // OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) - // } else { - // OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) - // } - OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) + if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) != null) { + OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) + } else { + OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) + } } @BeforeAll @JvmStatic fun setup() { ollama.start() - // ollama.commitToImage(NEW_IMAGE_NAME) + ollama.commitToImage(NEW_IMAGE_NAME) } @AfterAll @JvmStatic fun teardown() { - // ollama.commitToImage(NEW_IMAGE_NAME) + ollama.commitToImage(NEW_IMAGE_NAME) ollama.stop() } } From 984e97d67c843f9ac837e32453e61387c571a56c Mon Sep 17 00:00:00 2001 From: raulraja Date: Tue, 28 May 2024 16:55:13 +0200 Subject: [PATCH 09/12] Rewire tests container setup lifecycle --- .../ollama/tests/EnumClassificationTest.kt | 33 +++-- .../ollama/tests/EnumClassificationTest2.kt | 22 --- .../xef/ollama/tests/OllamaTests.kt | 127 +++++++++--------- .../xef/ollama/tests/models/Sentiment.kt | 3 - 4 files changed, 82 insertions(+), 103 deletions(-) delete mode 100644 core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt index 7a940ebcc..100c32374 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest.kt @@ -6,31 +6,28 @@ import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.Test class EnumClassificationTest : OllamaTests() { + @Test - fun `enum classification`() { + fun `positive sentiment`() { runBlocking { - val models = setOf(OllamaModels.LLama3_8B) - val sentiments = + val sentiment = ollama( - models = models, - prompt = "The sentiment of this text is positive.", + model = OllamaModels.Gemma2B, + prompt = "The context of the situation is very positive.", ) - expectSentiment(Sentiment.POSITIVE, sentiments, models) + assert(sentiment == Sentiment.POSITIVE) { "Expected POSITIVE but got $sentiment" } } } - companion object { - internal fun expectSentiment( - expected: Sentiment, - sentiments: List, - models: Set - ) { - assert(sentiments.size == models.size) { - "Expected ${models.size} results but got ${sentiments.size}" - } - sentiments.forEach { sentiment -> - assert(sentiment == expected) { "Expected $expected but got $sentiment" } - } + @Test + fun `negative sentiment`() { + runBlocking { + val sentiment = + ollama( + model = OllamaModels.LLama3_8B, + prompt = "The context of the situation is very negative.", + ) + assert(sentiment == Sentiment.NEGATIVE) { "Expected NEGATIVE but got $sentiment" } } } } diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt deleted file mode 100644 index 8cf08deb0..000000000 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/EnumClassificationTest2.kt +++ /dev/null @@ -1,22 +0,0 @@ -package com.xebia.functional.xef.ollama.tests - -import com.xebia.functional.xef.ollama.tests.EnumClassificationTest.Companion.expectSentiment -import com.xebia.functional.xef.ollama.tests.models.OllamaModels -import com.xebia.functional.xef.ollama.tests.models.Sentiment -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Test - -class EnumClassificationTest2 : OllamaTests() { - @Test - fun `enum classification 2`() { - runBlocking { - val models = setOf(OllamaModels.LLama3_8B) - val sentiments = - ollama( - models = models, - prompt = "The sentiment of this text is negative.", - ) - expectSentiment(Sentiment.NEGATIVE, sentiments, models) - } - } -} diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt index 581098282..470810073 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt @@ -1,15 +1,14 @@ package com.xebia.functional.xef.ollama.tests -import arrow.fx.coroutines.parMap -import com.xebia.functional.openai.generated.api.Chat +import com.github.dockerjava.api.model.Image import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel import com.xebia.functional.xef.AI import com.xebia.functional.xef.Config import com.xebia.functional.xef.OpenAI import io.github.oshai.kotlinlogging.KotlinLogging -import kotlinx.coroutines.Dispatchers +import java.util.concurrent.ConcurrentHashMap import org.junit.jupiter.api.AfterAll -import org.junit.jupiter.api.BeforeAll +import org.testcontainers.DockerClientFactory import org.testcontainers.ollama.OllamaContainer import org.testcontainers.utility.DockerImageName @@ -19,75 +18,83 @@ abstract class OllamaTests { companion object { private const val OLLAMA_IMAGE = "ollama/ollama:0.1.26" - private const val NEW_IMAGE_NAME = "ollama/ollama:test" - val ollama: OllamaContainer by lazy { - // check if the new image is already present otherwise pull the image - if (DockerImageName.parse(NEW_IMAGE_NAME).asCompatibleSubstituteFor(OLLAMA_IMAGE) != null) { - OllamaContainer(DockerImageName.parse(NEW_IMAGE_NAME)) + private val registeredContainers: MutableMap = ConcurrentHashMap() + + @PublishedApi + internal fun useModel(model: String): OllamaContainer = + if (registeredContainers.containsKey(model)) { + registeredContainers[model]!! } else { - OllamaContainer(DockerImageName.parse(OLLAMA_IMAGE)) + ollamaContainer(model) } - } - @BeforeAll - @JvmStatic - fun setup() { - ollama.start() - ollama.commitToImage(NEW_IMAGE_NAME) + private fun ollamaContainer(model: String, imageName: String = model): OllamaContainer { + if (registeredContainers.containsKey(model)) { + return registeredContainers[model]!! + } + // create the new image if it is not already a docker image + val listImagesCmd: List = + DockerClientFactory.lazyClient().listImagesCmd().withImageNameFilter(imageName).exec() + + val ollama = + if (listImagesCmd.isEmpty()) { + // ship container emoji: 🚒 + println("🐳 Creating a new Ollama container with $model image...") + val ollama = OllamaContainer(OLLAMA_IMAGE) + ollama.start() + println("🐳 Pulling $model image...") + ollama.execInContainer("ollama", "pull", model) + println("🐳 Committing $model image...") + ollama.commitToImage(imageName) + ollama.withReuse(true) + } else { + println("🐳 Using existing Ollama container with $model image...") + // Substitute the default Ollama image with our model variant + val ollama = + OllamaContainer( + DockerImageName.parse(imageName).asCompatibleSubstituteFor("ollama/ollama") + ) + .withReuse(true) + ollama.start() + ollama + } + println("🐳 Starting Ollama container with $model image...") + registeredContainers[model] = ollama + ollama.execInContainer("ollama", "run", model) + return ollama } @AfterAll @JvmStatic fun teardown() { - ollama.commitToImage(NEW_IMAGE_NAME) - ollama.stop() + registeredContainers.forEach { (model, container) -> + println("🐳 Stopping Ollama container for model $model") + container.stop() + } } } - suspend inline fun ollama( - models: Set, + protected suspend inline fun ollama( + model: String, prompt: String, - config: Config = Config(baseUrl = ollamaBaseUrl(), supportsLogitBias = false), - api: Chat = OpenAI(config = config, logRequests = true).chat, - ): List { - // pull all models - models.parMap(context = Dispatchers.IO) { model -> - logger.info { "🚒 Pulling model $model" } - val pullResult = ollama.execInContainer("ollama", "pull", model) - if (pullResult.exitCode != 0) { - logger.error { pullResult.stderr } - throw RuntimeException("Failed to pull model $model") - } - logger.info { pullResult.stdout } - logger.info { "🚒 Pulled $model" } - } - // run all models - models.parMap(context = Dispatchers.IO) { model -> - logger.info { "πŸš€ Starting model $model" } - val runResult = ollama.execInContainer("ollama", "run", model) - if (runResult.exitCode != 0) { - logger.error { runResult.stderr } - throw RuntimeException("Failed to run model $model") - } - logger.info { runResult.stdout } - println("πŸš€ Started $model") - } - // run inference on all models - return models.parMap(context = Dispatchers.IO) { model -> - logger.info { "πŸš€ Running inference on model $model" } - val result: A = - AI( - prompt = prompt, - config = config, - api = api, - model = CreateChatCompletionRequestModel.Custom(model), - ) - logger.info { "πŸš€ Inference on model $model: $result" } - result - } + ): A { + useModel(model) + val config = Config(supportsLogitBias = false, baseUrl = ollamaBaseUrl(model)) + val api = OpenAI(config = config, logRequests = true).chat + val result: A = + AI( + prompt = prompt, + config = config.copy(), + api = api, + model = CreateChatCompletionRequestModel.Custom(model), + ) + logger.info { "πŸš€ Inference on model $model: $result" } + return result } - fun ollamaBaseUrl(): String = - "http://${ollama.host}:${ollama.getMappedPort(ollama.exposedPorts.first())}/v1/" + fun ollamaBaseUrl(model: String): String { + val ollama = registeredContainers[model]!! + return "http://${ollama.host}:${ollama.getMappedPort(ollama.exposedPorts.first())}/v1/" + } } diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt index b83d008c4..f1e3374d5 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/models/Sentiment.kt @@ -6,7 +6,4 @@ import kotlinx.serialization.Serializable enum class Sentiment { POSITIVE, NEGATIVE, - NEUTRAL, - MIXED, - UNKNOWN } From 5c0eea39b3a82e87da99c9e3c31dca09d601f503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Raja=20Mart=C3=ADnez?= Date: Thu, 30 May 2024 09:46:28 +0200 Subject: [PATCH 10/12] Update core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Javier PΓ©rez Pacheco --- core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt index 7d9b678ae..c404c49de 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt @@ -50,7 +50,7 @@ fun OpenAI( val token = config.token ?: getenv(KEY_ENV_VAR) - ?: "" // throw AIError.Env.OpenAI(nonEmptyListOf("missing $KEY_ENV_VAR env + ?: "" // var")) val clientConfig: HttpClientConfig<*>.() -> Unit = { install(ContentNegotiation) { json(config.json) } From 543d4917ca1755060d97612545584cf9dbad2291 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Raja=20Mart=C3=ADnez?= Date: Thu, 30 May 2024 09:46:34 +0200 Subject: [PATCH 11/12] Update core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Javier PΓ©rez Pacheco --- .../kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt index 470810073..b3f18f7f1 100644 --- a/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt +++ b/core/src/jvmTest/kotlin/com/xebia/functional/xef/ollama/tests/OllamaTests.kt @@ -85,7 +85,7 @@ abstract class OllamaTests { val result: A = AI( prompt = prompt, - config = config.copy(), + config = config, api = api, model = CreateChatCompletionRequestModel.Custom(model), ) From b6fd9ff911a929b33fe0699ca5f8b1279ca73763 Mon Sep 17 00:00:00 2001 From: raulraja Date: Thu, 30 May 2024 07:48:12 +0000 Subject: [PATCH 12/12] Apply spotless formatting --- .../src/commonMain/kotlin/com/xebia/functional/xef/Config.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt index c404c49de..f96a6b75e 100644 --- a/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt +++ b/core/src/commonMain/kotlin/com/xebia/functional/xef/Config.kt @@ -47,10 +47,7 @@ fun OpenAI( httpClientConfig: ((HttpClientConfig<*>) -> Unit)? = null, logRequests: Boolean = false ): OpenAI { - val token = - config.token - ?: getenv(KEY_ENV_VAR) - ?: "" + val token = config.token ?: getenv(KEY_ENV_VAR) ?: "" // var")) val clientConfig: HttpClientConfig<*>.() -> Unit = { install(ContentNegotiation) { json(config.json) }