From c50a3bcaeb13e97615e1e42b12e2075d3a04c63c Mon Sep 17 00:00:00 2001 From: Peter Trifanov Date: Tue, 19 Oct 2021 12:09:27 +0300 Subject: [PATCH 1/3] Content-Encoding: gzip ### What's done: * Support in agent --- save-agent/build.gradle.kts | 1 + .../src/linuxX64Main/kotlin/org/cqfn/save/agent/Main.kt | 4 ++++ .../org/cqfn/save/orchestrator/docker/ContainerManager.kt | 1 + 3 files changed, 6 insertions(+) diff --git a/save-agent/build.gradle.kts b/save-agent/build.gradle.kts index 15ef9a1149..13aa6cff93 100644 --- a/save-agent/build.gradle.kts +++ b/save-agent/build.gradle.kts @@ -29,6 +29,7 @@ kotlin { implementation("io.ktor:ktor-client-core:${Versions.ktor}") implementation("io.ktor:ktor-client-curl:${Versions.ktor}") implementation("io.ktor:ktor-client-serialization:${Versions.ktor}") + implementation("io.ktor:ktor-client-encoding:${Versions.ktor}") implementation("org.jetbrains.kotlinx:kotlinx-serialization-properties:${Versions.serialization}") implementation("com.squareup.okio:okio-multiplatform:${Versions.okio}") implementation("org.jetbrains.kotlinx:kotlinx-datetime:${Versions.kotlinxDatetime}") diff --git a/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/Main.kt b/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/Main.kt index 8014caa081..7e1d134d4c 100644 --- a/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/Main.kt +++ b/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/Main.kt @@ -12,6 +12,7 @@ import org.cqfn.save.core.logging.logInfo import generated.SAVE_CLOUD_VERSION import io.ktor.client.HttpClient import io.ktor.client.features.HttpTimeout +import io.ktor.client.features.compression.ContentEncoding import io.ktor.client.features.json.JsonFeature import io.ktor.client.features.json.serializer.KotlinxSerializer import platform.posix.SIGTERM @@ -60,6 +61,9 @@ fun main() { install(HttpTimeout) { requestTimeoutMillis = config.requestTimeoutMillis } + install(ContentEncoding) { + gzip() + } } val saveAgent = SaveAgent(config, httpClient) runBlocking { diff --git a/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt index d3175fb69b..ef7fd0f971 100644 --- a/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt +++ b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/docker/ContainerManager.kt @@ -141,6 +141,7 @@ class ContainerManager(private val settings: DockerSettings) { val dockerFile = createTempFile(tmpDir.toPath()).toFile() dockerFile.writeText(dockerFileAsText) val hostIp = getHostIp("host.docker.internal") + log.debug("Resolved host IP as $hostIp, will add it to the container") val buildImageResultCallback: BuildImageResultCallback = try { dockerClient.buildImageCmd(dockerFile) .withBaseDirectory(tmpDir) From 90c019ba465cf7b59e299ec51625dfcf074bf5f5 Mon Sep 17 00:00:00 2001 From: Peter Trifanov Date: Thu, 13 Jan 2022 11:44:23 +0300 Subject: [PATCH 2/3] Encoding * Fixes after merge --- gradle/libs.versions.toml | 1 + save-agent/build.gradle.kts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 60d0f50b8c..694a130d1c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -85,6 +85,7 @@ jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module- ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-curl = { module = "io.ktor:ktor-client-curl", version.ref = "ktor" } ktor-client-serialization = { module = "io.ktor:ktor-client-serialization", version.ref = "ktor" } +ktor-client-encoding = { module = "io.ktor:ktor-client-encoding", version.ref = "ktor" } ktor-client-mock = { module = "io.ktor:ktor-client-mock", version.ref = "ktor" } # database diff --git a/save-agent/build.gradle.kts b/save-agent/build.gradle.kts index 0e736f8149..ef5dca0461 100644 --- a/save-agent/build.gradle.kts +++ b/save-agent/build.gradle.kts @@ -28,7 +28,7 @@ kotlin { implementation(libs.ktor.client.core) implementation(libs.ktor.client.curl) implementation(libs.ktor.client.serialization) -// implementation("io.ktor:ktor-client-encoding:${Versions.ktor}") + implementation(libs.ktor.client.encoding) implementation(libs.kotlinx.serialization.properties) implementation(libs.okio) implementation(libs.kotlinx.datetime) From e9b9ff874b82c6b12d516458acb5dab8a48b2ed1 Mon Sep 17 00:00:00 2001 From: Peter Trifanov Date: Thu, 13 Jan 2022 11:46:43 +0300 Subject: [PATCH 3/3] [skip ci] Encoding * Experiments with deflating with zlib --- build.gradle.kts | 1 + .../org/cqfn/save/agent/utils/FileUtils.kt | 38 +++++++++++++++++++ .../kotlin/org/cqfn/save/utils/Utils.kt | 7 ++++ .../filters/GzipDecodingFilter.kt | 12 ++++++ 4 files changed, 58 insertions(+) create mode 100644 save-cloud-common/src/commonMain/kotlin/org/cqfn/save/utils/Utils.kt create mode 100644 save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/filters/GzipDecodingFilter.kt diff --git a/build.gradle.kts b/build.gradle.kts index 3b24a98cfc..3a9ef93bd4 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -11,6 +11,7 @@ plugins { alias(libs.plugins.versions.plugin) alias(libs.plugins.talaiot.base) alias(libs.plugins.liquibase.gradle) + id("com.louiscad.complete-kotlin") version "1.1.0" } val profile = properties.getOrDefault("save.profile", "dev") as String diff --git a/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/utils/FileUtils.kt b/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/utils/FileUtils.kt index 180c0225f5..0fca2697c2 100644 --- a/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/utils/FileUtils.kt +++ b/save-agent/src/linuxX64Main/kotlin/org/cqfn/save/agent/utils/FileUtils.kt @@ -7,6 +7,44 @@ package org.cqfn.save.agent.utils import okio.FileNotFoundException import okio.FileSystem import okio.Path.Companion.toPath +import platform.zlib.* + +import kotlinx.cinterop.UByteVar +import kotlinx.cinterop.allocArray +import kotlinx.cinterop.cValuesOf +import kotlinx.cinterop.cstr +import kotlinx.cinterop.memScoped +import kotlinx.cinterop.pointed +import kotlinx.cinterop.ptr +import kotlinx.cinterop.readBytes +import kotlinx.cinterop.reinterpret +import kotlinx.cinterop.value + +/** + * @param s + * @return + */ +fun deflate(s: String): ByteArray = memScoped { + // val defstream: z_stream = z_stream() + // defstream.zalloc = Z_NULL + // defstream.zfree = Z_NULL + // defstream.avail_in = s.length.toUInt() // size of input + // defstream.next_in = UByteVarOf(s.cstr.ptr.rawValue).ptr + // defstream.avail_in = s.length.toUInt() // size of input + val out = allocArray(s.length) + // defstream.next_out = UByteVarOf(out.rawValue).ptr + // deflateInit(defstream.ptr, Z_BEST_COMPRESSION) + // platform.zlib.deflate(defstream.ptr, Z_FINISH) + // deflateEnd(defstream.ptr) + val destLen = cValuesOf(s.length).ptr + compress( + out, + destLen.reinterpret(), + s.cstr.ptr.reinterpret(), + s.length.toULong() + ) + return@memScoped out.readBytes(destLen.pointed.value) +} /** * Read file as a list of strings diff --git a/save-cloud-common/src/commonMain/kotlin/org/cqfn/save/utils/Utils.kt b/save-cloud-common/src/commonMain/kotlin/org/cqfn/save/utils/Utils.kt new file mode 100644 index 0000000000..7304a7353a --- /dev/null +++ b/save-cloud-common/src/commonMain/kotlin/org/cqfn/save/utils/Utils.kt @@ -0,0 +1,7 @@ +package org.cqfn.save.utils + +/** + * @param default + * @return + */ +fun String?.ifNullOrEmpty(default: () -> String) = (this ?: "").ifBlank(default) diff --git a/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/filters/GzipDecodingFilter.kt b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/filters/GzipDecodingFilter.kt new file mode 100644 index 0000000000..c9862a0e5f --- /dev/null +++ b/save-orchestrator/src/main/kotlin/org/cqfn/save/orchestrator/filters/GzipDecodingFilter.kt @@ -0,0 +1,12 @@ +package org.cqfn.save.orchestrator.filters + +import org.springframework.web.server.ServerWebExchange +import org.springframework.web.server.WebFilter +import org.springframework.web.server.WebFilterChain +import reactor.core.publisher.Mono + +class GzipDecodingFilter : WebFilter { + override fun filter(exchange: ServerWebExchange, chain: WebFilterChain): Mono { + TODO("Not yet implemented") + } +}