diff --git a/.github/workflows/build-base-images.yaml b/.github/workflows/build-base-images.yaml index f4c787e08a..87c4a9e0fd 100644 --- a/.github/workflows/build-base-images.yaml +++ b/.github/workflows/build-base-images.yaml @@ -4,6 +4,12 @@ on: schedule: - cron: '0 4 * * 1' workflow_dispatch: + inputs: + branch: + type: string + default: master + description: Branch to build images from + required: false jobs: build_base_images: @@ -42,6 +48,10 @@ jobs: base_image_tag: '3.10' steps: - uses: actions/checkout@v3 + - if: github.event_name == 'workflow_dispatch' + name: Prepare to build from branch + run: | + git checkout origin/${{ inputs.branch }} - uses: docker/login-action@v2 with: registry: ghcr.io diff --git a/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/DockerStackConfiguration.kt b/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/DockerStackConfiguration.kt index 31ad35bd01..ecd5dfcb2e 100644 --- a/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/DockerStackConfiguration.kt +++ b/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/DockerStackConfiguration.kt @@ -17,6 +17,7 @@ import java.nio.file.Files import java.nio.file.Paths const val MYSQL_STARTUP_DELAY_MILLIS = 30_000L +const val KAFKA_STARTUP_DELAY_MILLIS = 5_000L /** * @param profile deployment profile, used, for example, to start SQL database in dev profile only @@ -53,7 +54,28 @@ fun Project.createStackDeployTask(profile: String) { | environment: | - "MYSQL_ROOT_PASSWORD=123" | - "MYSQL_DATABASE=save_cloud" - """.trimMargin() + | zookeeper: + | image: confluentinc/cp-zookeeper:latest + | environment: + | ZOOKEEPER_CLIENT_PORT: 2181 + | ZOOKEEPER_TICK_TIME: 2000 + | ports: + | - 22181:2181 + | + | kafka: + | image: confluentinc/cp-kafka:latest + | depends_on: + | - zookeeper + | ports: + | - 29092:29092 + | environment: + | KAFKA_BROKER_ID: 1 + | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092 + | KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT + | KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT + | KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + """.trimMargin() } else if (profile == "dev" && it.trim().startsWith("logging:")) { "" } else { @@ -106,9 +128,11 @@ fun Project.createStackDeployTask(profile: String) { Files.createDirectories(configsDir.resolve("orchestrator")) Files.createDirectories(configsDir.resolve("preprocessor")) } - description = "Deploy to docker swarm. If swarm contains more than one node, some registry for built images is required." + description = + "Deploy to docker swarm. If swarm contains more than one node, some registry for built images is required." // this command puts env variables into compose file - val composeCmd = "docker-compose -f ${rootProject.buildDir}/docker-compose.yaml --env-file ${rootProject.buildDir}/.env config" + val composeCmd = + "docker-compose -f ${rootProject.buildDir}/docker-compose.yaml --env-file ${rootProject.buildDir}/.env config" val stackCmd = "docker stack deploy --compose-file -" + if (useOverride && composeOverride.exists()) { " --compose-file ${composeOverride.canonicalPath}" @@ -125,12 +149,13 @@ fun Project.createStackDeployTask(profile: String) { } tasks.register("stopDockerStack") { - description = "Completely stop all services in docker swarm. NOT NEEDED FOR REDEPLOYING! Use only to explicitly stop everything." + description = + "Completely stop all services in docker swarm. NOT NEEDED FOR REDEPLOYING! Use only to explicitly stop everything." commandLine("docker", "stack", "rm", "save") } // in case you are running it on MAC, first do the following: docker pull --platform linux/x86_64 mysql - tasks.register("startMysqlDb") { + tasks.register("startMysqlDbService") { dependsOn("generateComposeFile") doFirst { logger.lifecycle("Running the following command: [docker-compose --file $buildDir/docker-compose.yaml up -d mysql]") @@ -138,12 +163,29 @@ fun Project.createStackDeployTask(profile: String) { commandLine("docker-compose", "--file", "$buildDir/docker-compose.yaml", "up", "-d", "mysql") errorOutput = ByteArrayOutputStream() doLast { - if (!errorOutput.toString().contains(" is up-to-date")) { - logger.lifecycle("Waiting $MYSQL_STARTUP_DELAY_MILLIS millis for mysql to start") - Thread.sleep(MYSQL_STARTUP_DELAY_MILLIS) // wait for mysql to start, can be manually increased when needed - } + logger.lifecycle("Waiting $MYSQL_STARTUP_DELAY_MILLIS millis for mysql to start") + Thread.sleep(MYSQL_STARTUP_DELAY_MILLIS) // wait for mysql to start, can be manually increased when needed + } + } + tasks.named("liquibaseUpdate") { + mustRunAfter("startMysqlDbService") + } + tasks.register("startMysqlDb") { + dependsOn("liquibaseUpdate") + dependsOn("startMysqlDbService") + } + + tasks.register("startKafka") { + dependsOn("generateComposeFile") + doFirst { + logger.lifecycle("Running the following command: [docker-compose --file $buildDir/docker-compose.yaml up -d kafka]") + } + errorOutput = ByteArrayOutputStream() + commandLine("docker-compose", "--file", "$buildDir/docker-compose.yaml", "up", "-d", "kafka") + doLast { + logger.lifecycle("Waiting $KAFKA_STARTUP_DELAY_MILLIS millis for kafka to start") + Thread.sleep(KAFKA_STARTUP_DELAY_MILLIS) // wait for kafka to start, can be manually increased when needed } - finalizedBy("liquibaseUpdate") } tasks.register("restartMysqlDb") { @@ -152,10 +194,26 @@ fun Project.createStackDeployTask(profile: String) { finalizedBy("startMysqlDb") } + tasks.register("restartKafka") { + dependsOn("generateComposeFile") + commandLine("docker-compose", "--file", "$buildDir/docker-compose.yaml", "rm", "--force", "kafka") + commandLine("docker-compose", "--file", "$buildDir/docker-compose.yaml", "rm", "--force", "zookeeper") + finalizedBy("startKafka") + } + tasks.register("deployLocal") { dependsOn(subprojects.flatMap { it.tasks.withType() }) dependsOn("startMysqlDb") - commandLine("docker-compose", "--file", "$buildDir/docker-compose.yaml", "up", "-d", "orchestrator", "backend", "preprocessor") + commandLine( + "docker-compose", + "--file", + "$buildDir/docker-compose.yaml", + "up", + "-d", + "orchestrator", + "backend", + "preprocessor" + ) } val componentName = findProperty("save.component") as String? @@ -166,7 +224,8 @@ fun Project.createStackDeployTask(profile: String) { "and it should be a name of one of gradle subprojects. If component name is `save-backend`, then `save-frontend` will be built too" + " and bundled into save-backend image." require(componentName in allprojects.map { it.name }) { "Component name should be one of gradle subproject names, but was [$componentName]" } - val buildTask: TaskProvider = project(componentName).tasks.named("bootBuildImage") + val buildTask: TaskProvider = + project(componentName).tasks.named("bootBuildImage") dependsOn(buildTask) val serviceName = when (componentName) { "save-backend", "save-orchestrator", "save-preprocessor" -> "save_${componentName.substringAfter("save-")}" diff --git a/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/VersioningConfiguration.kt b/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/VersioningConfiguration.kt index 04a1d0b214..b2247da34c 100644 --- a/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/VersioningConfiguration.kt +++ b/buildSrc/src/main/kotlin/com/saveourtool/save/buildutils/VersioningConfiguration.kt @@ -66,7 +66,7 @@ fun Project.configureVersioning() { */ fun Project.versionForDockerImages(): String = (project.findProperty("dockerTag") as String? ?: version.toString()) - .replace(Regex("[^._\\-a-z0-9]"), "-") + .replace(Regex("[^._\\-a-zA-Z0-9]"), "-") /** * Register task that reads version of save-cli, either from project property, or from Versions, or latest diff --git a/diktat-analysis.yml b/diktat-analysis.yml index 441c713f51..8d14c5d906 100644 --- a/diktat-analysis.yml +++ b/diktat-analysis.yml @@ -92,7 +92,7 @@ - name: TOO_LONG_FUNCTION enabled: true configuration: - maxFunctionLength: 35 # max length of function + maxFunctionLength: 55 # max length of function isIncludeHeader: false # count function's header - name: TOO_MANY_PARAMETERS enabled: true diff --git a/docker-compose.yaml b/docker-compose.yaml index 7839583f48..febae6d1cc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -84,7 +84,7 @@ services: - "prometheus-job=api-gateway" logging: *loki-logging-jvm prometheus: - image: prom/prometheus:v2.37.0 + image: prom/prometheus:v2.38.0 user: root # to access host's docker socket for service discovery, see https://groups.google.com/g/prometheus-users/c/EuEW0qRzXvg/m/0aqKh_ZABQAJ ports: - "9090:9090" @@ -101,7 +101,7 @@ services: constraints: - "node.role==manager" grafana: - image: grafana/grafana:9.0.7 + image: grafana/grafana:9.1.0 ports: - "9100:3000" volumes: diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 77d4cb5e10..450bacd93c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,6 +12,7 @@ spring-boot = "2.7.2" spring-security = "5.7.2" spring-cloud = "3.1.3" spring-cloud-kubernetes = "2.1.3" +kafka-client = "3.2.0" junit = "5.9.0" diktat = "1.2.3" detekt = "1.21.0" @@ -92,9 +93,14 @@ spring-cloud-starter-gateway = { module = "org.springframework.cloud:spring-clou spring-cloud-starter-kubernetes-client-config = { module = "org.springframework.cloud:spring-cloud-starter-kubernetes-client-config", version.ref = "spring-cloud-kubernetes" } spring-boot-starter-oauth2-client = { module = "org.springframework.boot:spring-boot-starter-oauth2-client" } spring-context-indexer = { module = "org.springframework:spring-context-indexer", version.ref = "spring" } +spring-kafka = {module = "org.springframework.kafka:spring-kafka"} +spring-kafka-test = {module = "org.springframework.kafka:spring-kafka-test"} spring-web = { module = "org.springframework:spring-web", version.ref = "spring" } jackson-module-kotlin = { module = "com.fasterxml.jackson.module:jackson-module-kotlin" } +kafka-clients = {module = "org.apache.kafka:kafka-clients", version.ref = "kafka-client"} +kafka212 = {module = "org.apache.kafka:kafka_2.12", version.ref = "kafka-client"} + springdoc-openapi-ui = { module = "org.springdoc:springdoc-openapi-ui", version.ref = "springdoc" } springdoc-openapi-webflux-ui = { module = "org.springdoc:springdoc-openapi-webflux-ui", version.ref = "springdoc" } springdoc-openapi-security = { module = "org.springdoc:springdoc-openapi-security", version.ref = "springdoc" } @@ -107,6 +113,8 @@ ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-curl = { module = "io.ktor:ktor-client-curl", version.ref = "ktor" } ktor-client-logging = { module = "io.ktor:ktor-client-logging", version.ref = "ktor" } ktor-client-serialization = { module = "io.ktor:ktor-client-serialization", version.ref = "ktor" } +ktor-client-content-negotiation = { module = "io.ktor:ktor-client-content-negotiation", version.ref = "ktor" } +ktor-serialization-kotlinx-json = { module = "io.ktor:ktor-serialization-kotlinx-json", version.ref = "ktor" } ktor-client-mock = { module = "io.ktor:ktor-client-mock", version.ref = "ktor" } # database diff --git a/save-agent/README.md b/save-agent/README.md index 37fc07cf8c..e16632f87b 100644 --- a/save-agent/README.md +++ b/save-agent/README.md @@ -1,5 +1,7 @@ # Building Read the [official docs](https://github.com/JetBrains/kotlin/tree/master/kotlin-native#kotlinnative) and install all dependencies listed in [Prerequisites](https://github.com/JetBrains/kotlin/tree/master/kotlin-native#building-from-source) section. -On windows you'll also need to install msys2 and run `pacman -S mingw-w64-x86_64-curl` to have libcurl for ktor-client. -On ubuntu install `libcurl4-openssl-dev` for ktor client. \ No newline at end of file +On Windows you'll also need to install msys2 and run `pacman -S mingw-w64-x86_64-curl` to have libcurl for ktor-client. +On ubuntu install `libcurl4-openssl-dev` for ktor client. + +`save-agent` also requires `unzip` to be present on `$PATH`. diff --git a/save-agent/build.gradle.kts b/save-agent/build.gradle.kts index f95f789999..ab2836a56c 100644 --- a/save-agent/build.gradle.kts +++ b/save-agent/build.gradle.kts @@ -33,7 +33,9 @@ kotlin { implementation(libs.save.reporters) implementation(libs.ktor.client.core) implementation(libs.ktor.client.curl) - implementation(libs.ktor.client.serialization) + implementation(libs.ktor.client.content.negotiation) + implementation(libs.ktor.serialization.kotlinx.json) + implementation(libs.ktor.client.logging) implementation(libs.kotlinx.serialization.properties) implementation(libs.okio) implementation(libs.kotlinx.datetime) diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt index 281fb18cec..c83b42178e 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/AgentConfiguration.kt @@ -24,6 +24,7 @@ import kotlinx.serialization.Serializable * @property cliCommand a command that agent will use to run SAVE cli * @property debug whether debug logging should be enabled * @property retry configuration for HTTP request retries + * @property testSuitesDir directory where tests and additional files need to be stored into * @property logFilePath path to logs of save-cli execution * @property save additional configuration for save-cli */ @@ -37,6 +38,7 @@ data class AgentConfiguration( val retry: RetryConfig, val debug: Boolean = false, val cliCommand: String, + val testSuitesDir: String, val logFilePath: String = "logs.txt", val save: SaveCliConfig = SaveCliConfig(), ) { @@ -68,12 +70,14 @@ data class HeartbeatConfig( * @property additionalDataEndpoint endpoint to post additional data (version etc.) to * @property executionDataEndpoint endpoint to post execution data to * @property filesEndpoint endpoint to post debug info to + * @property testSourceSnapshotEndpoint endpoint to download test source snapshots from */ @Serializable data class BackendConfig( val url: String, val additionalDataEndpoint: String = "internal/saveAgentVersion", val executionDataEndpoint: String = "internal/saveTestResult", + val testSourceSnapshotEndpoint: String = "/internal/test-suites-sources/download-snapshot-by-execution-id", val filesEndpoint: String = "internal/files", ) diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Main.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Main.kt index 3a8b2a5cd3..a58e588489 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Main.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Main.kt @@ -4,18 +4,24 @@ package com.saveourtool.save.agent +import com.saveourtool.save.agent.utils.ktorLogger import com.saveourtool.save.agent.utils.logDebugCustom import com.saveourtool.save.agent.utils.logInfoCustom +import com.saveourtool.save.agent.utils.markAsExecutable import com.saveourtool.save.agent.utils.readProperties import com.saveourtool.save.core.config.LogType +import com.saveourtool.save.core.logging.describe import com.saveourtool.save.core.logging.logType import generated.SAVE_CLOUD_VERSION import generated.SAVE_CORE_VERSION import io.ktor.client.HttpClient -import io.ktor.client.plugins.HttpTimeout -import io.ktor.client.plugins.json.JsonPlugin -import io.ktor.client.plugins.kotlinx.serializer.KotlinxSerializer +import io.ktor.client.plugins.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.plugins.logging.* +import io.ktor.serialization.kotlinx.json.* +import okio.FileSystem +import okio.Path.Companion.toPath import platform.posix.* import kotlinx.cinterop.staticCFunction @@ -42,6 +48,8 @@ internal val json = Json { } } +internal val fs = FileSystem.SYSTEM + @OptIn(ExperimentalSerializationApi::class) fun main() { val config: AgentConfiguration = Properties.decodeFromStringMap( @@ -50,24 +58,14 @@ fun main() { logType.set(if (config.debug) LogType.ALL else LogType.WARN) logDebugCustom("Instantiating save-agent version $SAVE_CLOUD_VERSION with config $config") - platform.posix.chmod( - "save-$SAVE_CORE_VERSION-linuxX64.kexe", - (S_IRUSR or S_IWUSR or S_IXUSR or S_IRGRP or S_IROTH).toUInt() - ) + "save-$SAVE_CORE_VERSION-linuxX64.kexe".toPath().markAsExecutable() signal(SIGTERM, staticCFunction { logInfoCustom("Agent is shutting down because SIGTERM has been received") exit(1) }) - val httpClient = HttpClient { - install(JsonPlugin) { - serializer = KotlinxSerializer(json) - } - install(HttpTimeout) { - requestTimeoutMillis = config.requestTimeoutMillis - } - } + val httpClient = configureHttpClient(config) runBlocking { // Launch in a new scope, because we cancel the scope on graceful termination, @@ -81,3 +79,28 @@ fun main() { } logInfoCustom("Agent is shutting down") } + +@Suppress("FLOAT_IN_ACCURATE_CALCULATIONS", "MagicNumber") +private fun configureHttpClient(agentConfiguration: AgentConfiguration) = HttpClient { + install(ContentNegotiation) { + json(json = json) + } + install(HttpTimeout) { + requestTimeoutMillis = agentConfiguration.requestTimeoutMillis + } + install(HttpRequestRetry) { + retryOnException(maxRetries = agentConfiguration.retry.attempts) + retryOnServerErrors(maxRetries = agentConfiguration.retry.attempts) + exponentialDelay(base = agentConfiguration.retry.initialRetryMillis / 1000.0) + modifyRequest { + if (retryCount > 1) { + val reason = response?.status ?: cause?.describe() ?: "Unknown reason" + logDebugCustom("Retrying request: attempt #$retryCount, reason: $reason") + } + } + } + install(Logging) { + logger = ktorLogger + level = if (agentConfiguration.debug) LogLevel.ALL else LogLevel.INFO + } +} diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt new file mode 100644 index 0000000000..729dfe5e7f --- /dev/null +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/Requests.kt @@ -0,0 +1,110 @@ +/** + * Utilities to perform requests to other services of save-cloud + */ + +package com.saveourtool.save.agent + +import com.saveourtool.save.agent.utils.* +import com.saveourtool.save.agent.utils.extractZipTo +import com.saveourtool.save.agent.utils.markAsExecutable +import com.saveourtool.save.agent.utils.unzipIfRequired +import com.saveourtool.save.agent.utils.writeToFile +import com.saveourtool.save.core.logging.logWarn +import com.saveourtool.save.core.utils.runIf +import com.saveourtool.save.domain.FileKey + +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.http.* +import okio.Path +import okio.Path.Companion.toPath + +/** + * Download test source snapshots for execution [executionId] into [target] + * + * @param config + * @param target + * @param executionId + * @return result + */ +internal suspend fun SaveAgent.downloadTestResources(config: BackendConfig, target: Path, executionId: String): Result = runCatching { + val result = httpClient.downloadTestResources(config, executionId) + if (updateStateBasedOnBackendResponse(result)) { + return@runCatching + } + + val bytes = result.getOrThrow() + .readByteArrayOrThrowIfEmpty { + error("Not found any tests for execution $executionId") + } + val pathToArchive = "archive.zip".toPath() + logDebugCustom("Writing downloaded archive of size ${bytes.size} into $pathToArchive") + bytes.writeToFile(pathToArchive) + fs.createDirectories(target, mustCreate = false) + pathToArchive.extractZipTo(target) + fs.delete(pathToArchive, mustExist = true) + logDebugCustom("Extracted archive into $target and deleted $pathToArchive") +} + +/** + * Download additional resources from [additionalResourcesAsString] into [targetDirectory] + * + * @param baseUrl + * @param targetDirectory + * @param additionalResourcesAsString + * @param executionId + * @return result + */ +internal suspend fun SaveAgent.downloadAdditionalResources( + baseUrl: String, + targetDirectory: Path, + additionalResourcesAsString: String, + executionId: String, +) = runCatching { + FileKey.parseList(additionalResourcesAsString) + .map { fileKey -> + val result = httpClient.downloadFile( + "$baseUrl/internal/files/download?executionId=$executionId", + fileKey + ) + if (updateStateBasedOnBackendResponse(result)) { + return@runCatching + } + + val fileContentBytes = result.getOrThrow() + .readByteArrayOrThrowIfEmpty { + error("Couldn't download file $fileKey: content is empty") + } + val targetFile = targetDirectory / fileKey.name + fileContentBytes.writeToFile(targetFile) + fileKey to targetFile + } + .onEach { (fileKey, pathToFile) -> + pathToFile.markAsExecutable() + logDebugCustom( + "Downloaded $fileKey into ${fs.canonicalize(pathToFile)}" + ) + } + .map { (_, pathToFile) -> + unzipIfRequired(pathToFile) + } + .ifEmpty { + logWarn("Not found any additional files for execution \$id") + emptyList() + } +} + +private suspend fun HttpClient.downloadTestResources(config: BackendConfig, executionId: String) = download( + url = "${config.url}${config.testSourceSnapshotEndpoint}?executionId=$executionId", + body = null, +) + +private suspend fun HttpClient.downloadFile(url: String, fileKey: FileKey): Result = download( + url = url, + body = fileKey, +) + +private suspend fun HttpResponse.readByteArrayOrThrowIfEmpty(exceptionSupplier: ByteArray.() -> Nothing) = + body().runIf({ isEmpty() }, exceptionSupplier) diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt index 66d980119d..2f36f9b06b 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/SaveAgent.kt @@ -2,16 +2,16 @@ package com.saveourtool.save.agent -import com.saveourtool.save.agent.utils.logDebugCustom -import com.saveourtool.save.agent.utils.logErrorCustom -import com.saveourtool.save.agent.utils.logInfoCustom +import com.saveourtool.save.agent.utils.* import com.saveourtool.save.agent.utils.readFile +import com.saveourtool.save.agent.utils.requiredEnv import com.saveourtool.save.agent.utils.sendDataToBackend import com.saveourtool.save.core.logging.describe import com.saveourtool.save.core.plugin.Plugin import com.saveourtool.save.core.result.CountWarnings import com.saveourtool.save.core.utils.ExecutionResult import com.saveourtool.save.core.utils.ProcessBuilder +import com.saveourtool.save.core.utils.runIf import com.saveourtool.save.domain.TestResultDebugInfo import com.saveourtool.save.plugins.fix.FixPlugin import com.saveourtool.save.reporter.Report @@ -25,7 +25,6 @@ import io.ktor.client.request.* import io.ktor.client.request.forms.* import io.ktor.client.statement.* import io.ktor.http.* -import io.ktor.util.* import io.ktor.utils.io.core.* import okio.FileSystem import okio.Path.Companion.toPath @@ -33,14 +32,7 @@ import okio.buffer import kotlin.native.concurrent.AtomicLong import kotlin.native.concurrent.AtomicReference -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.Job -import kotlinx.coroutines.cancel -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay -import kotlinx.coroutines.launch -import kotlinx.coroutines.newSingleThreadContext +import kotlinx.coroutines.* import kotlinx.datetime.Clock import kotlinx.serialization.decodeFromString import kotlinx.serialization.json.Json @@ -52,16 +44,17 @@ import kotlinx.serialization.modules.subclass * A main class for SAVE Agent * @property config * @property coroutineScope a [CoroutineScope] to launch other jobs + * @property httpClient */ @Suppress("AVOID_NULL_CHECKS") -class SaveAgent(internal val config: AgentConfiguration, - private val httpClient: HttpClient, +class SaveAgent(private val config: AgentConfiguration, + internal val httpClient: HttpClient, private val coroutineScope: CoroutineScope, ) { /** - * The current [AgentState] of this agent + * The current [AgentState] of this agent. Initial value corresponds to the period when agent needs to finish its configuration. */ - val state = AtomicReference(AgentState.STARTING) + val state = AtomicReference(AgentState.BUSY) // fixme (limitation of old MM): can't use atomic reference to Instant here, because when using `Clock.System.now()` as an assigned value // Kotlin throws `kotlin.native.concurrent.InvalidMutabilityException: mutation attempt of frozen kotlinx.datetime.Instant...` @@ -86,7 +79,28 @@ class SaveAgent(internal val config: AgentConfiguration, fun start(): Job { logInfoCustom("Starting agent") coroutineScope.launch(backgroundContext) { + state.value = AgentState.BUSY sendDataToBackend { saveAdditionalData() } + + logDebugCustom("Will now download tests") + val executionId = requiredEnv("EXECUTION_ID") + val targetDirectory = config.testSuitesDir.toPath() + downloadTestResources(config.backend, targetDirectory, executionId).runIf({ isFailure }) { + logErrorCustom("Unable to download tests for execution $executionId: ${exceptionOrNull()?.describe()}") + state.value = AgentState.CRASHED + return@launch + } + logInfoCustom("Downloaded all tests for execution $executionId to $targetDirectory") + + logDebugCustom("Will now download additional resources") + val additionalFilesList = requiredEnv("ADDITIONAL_FILES_LIST") + downloadAdditionalResources(config.backend.url, targetDirectory, additionalFilesList, executionId).runIf({ isFailure }) { + logErrorCustom("Unable to download resources for execution $executionId based on list [$additionalFilesList]: ${exceptionOrNull()?.describe()}") + state.value = AgentState.CRASHED + return@launch + } + + state.value = AgentState.STARTING } return coroutineScope.launch { startHeartbeats(this) } } @@ -289,7 +303,6 @@ class SaveAgent(internal val config: AgentConfiguration, /** * @param byteArray byte array with logs of CLI execution progress that will be sent in a message */ - @OptIn(InternalAPI::class) private suspend fun sendLogs(byteArray: ByteArray): HttpResponse = httpClient.post { url("${config.orchestratorUrl}/executionLogs") diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/FileUtils.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/FileUtils.kt index 9056f1f736..2d27d57edb 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/FileUtils.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/FileUtils.kt @@ -4,9 +4,54 @@ package com.saveourtool.save.agent.utils +import com.saveourtool.save.agent.fs +import com.saveourtool.save.core.files.findAllFilesMatching import okio.FileNotFoundException import okio.FileSystem +import okio.Path import okio.Path.Companion.toPath +import platform.posix.S_IRGRP +import platform.posix.S_IROTH +import platform.posix.S_IRUSR +import platform.posix.S_IWUSR +import platform.posix.S_IXUSR + +/** + * Extract path as ZIP archive to provided directory + * + * @param targetPath + */ +internal fun Path.extractZipTo(targetPath: Path) { + require(fs.metadata(targetPath).isDirectory) + logDebugCustom("Unzip ${fs.canonicalize(this)} into ${fs.canonicalize(targetPath)}") + platform.posix.system("unzip $this -d $targetPath") +} + +/** + * Write content of [this] into a file [file] + * + * @receiver [ByteArray] to be written into a file + * @param file target [Path] + * @param mustCreate will be passed to Okio's [FileSystem.write] + */ +internal fun ByteArray.writeToFile(file: Path, mustCreate: Boolean = true) { + fs.write( + file = file, + mustCreate = mustCreate, + ) { + write(this@writeToFile).flush() + } +} + +/** + * Mark [this] file as executable. Sets permissions to rwxr--r-- + */ +internal fun Path.markAsExecutable() { + platform.posix.chmod( + this.toString(), + (S_IRUSR or S_IWUSR or S_IXUSR or S_IRGRP or S_IROTH).toUInt() + ) +} /** * Read file as a list of strings @@ -37,3 +82,23 @@ internal fun readProperties(filePath: String): Map = readFile(fi it.first() to it.last() } } + +/** + * @param pathToFile + */ +internal fun unzipIfRequired( + pathToFile: Path, +) { + // FixMe: for now support only .zip files + if (pathToFile.name.endsWith(".zip")) { + pathToFile.extractZipTo(pathToFile.parent!!) + // fixme: need to store information about isExecutable in Execution (FileKey) + pathToFile.parent!!.findAllFilesMatching { + if (fs.metadata(it).isRegularFile) { + it.markAsExecutable() + } + true + } + fs.delete(pathToFile, mustExist = true) + } +} diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/HttpUtils.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/HttpUtils.kt index 942961ac78..2c2ae93f05 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/HttpUtils.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/HttpUtils.kt @@ -5,55 +5,61 @@ package com.saveourtool.save.agent.utils import com.saveourtool.save.agent.AgentState -import com.saveourtool.save.agent.RetryConfig import com.saveourtool.save.agent.SaveAgent +import io.ktor.client.* +import io.ktor.client.request.* import io.ktor.client.statement.HttpResponse -import io.ktor.http.HttpStatusCode +import io.ktor.http.* -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay +/** + * @param result + * @return true if [this] agent's state has been updated to reflect problems with [result] + */ +internal fun SaveAgent.updateStateBasedOnBackendResponse( + result: Result +) = if (result.isSuccess && result.getOrNull()?.status != HttpStatusCode.OK) { + state.value = AgentState.BACKEND_FAILURE + true +} else if (result.isFailure) { + state.value = AgentState.BACKEND_UNREACHABLE + true +} else { + false +} /** - * Attempt to send execution data to backend, will retry several times, while increasing delay 2 times on each iteration. + * Attempt to send execution data to backend. * * @param requestToBackend + * @return a [Result] wrapping response */ internal suspend fun SaveAgent.sendDataToBackend( requestToBackend: suspend () -> HttpResponse -): Unit = sendWithRetries(config.retry, requestToBackend) { result, attempt -> - val reason = if (result.isSuccess && result.getOrNull()?.status != HttpStatusCode.OK) { +): Result = runCatching { requestToBackend() }.apply { + val reason = if (isSuccess && getOrNull()?.status != HttpStatusCode.OK) { state.value = AgentState.BACKEND_FAILURE - "Backend returned status ${result.getOrNull()?.status}" + "Backend returned status ${getOrNull()?.status}" } else { state.value = AgentState.BACKEND_UNREACHABLE - "Backend is unreachable, ${result.exceptionOrNull()?.message}" + "Backend is unreachable, ${exceptionOrNull()?.message}" } - logErrorCustom("Cannot post data (x${attempt + 1}), will retry in ${config.retry.initialRetryMillis} ms. Reason: $reason") + logErrorCustom("Cannot send data to backed: $reason") } /** - * @param retryConfig - * @param request - * @param onError + * Perform a POST request to [url] (optionally with body [body] that will be serialized as JSON), + * accepting application/octet-stream and return result wrapping [HttpResponse] + * + * @param url + * @param body + * @return result wrapping [HttpResponse] */ -@Suppress("TYPE_ALIAS") -internal suspend fun sendWithRetries( - retryConfig: RetryConfig, - request: suspend () -> HttpResponse, - onError: (Result, attempt: Int) -> Unit, -): Unit = coroutineScope { - var retryInterval = retryConfig.initialRetryMillis - repeat(retryConfig.attempts) { attempt -> - val result = runCatching { - request() - } - if (result.isSuccess && result.getOrNull()?.status == HttpStatusCode.OK) { - return@coroutineScope - } else { - onError(result, attempt) - delay(retryInterval) - retryInterval *= 2 - } +internal suspend fun HttpClient.download(url: String, body: Any?): Result = runCatching { + post { + url(url) + contentType(ContentType.Application.Json) + accept(ContentType.Application.OctetStream) + body?.let { setBody(it) } } } diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/LinuxUtils.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/LinuxUtils.kt new file mode 100644 index 0000000000..ebe651e239 --- /dev/null +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/LinuxUtils.kt @@ -0,0 +1,19 @@ +/** + * Utilities to work with Linux-specific calls + */ + +package com.saveourtool.save.agent.utils + +import platform.posix.getenv + +import kotlinx.cinterop.toKString + +/** + * Get value of environment variable [name] or throw if it is not set. + * + * @param name name of the environment variable + * @return value of the environment variable + */ +internal fun requiredEnv(name: String): String = requireNotNull(getenv(name)) { + "Environment variable $name is not set but is required" +}.toKString() diff --git a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/Logging.kt b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/Logging.kt index 6692621a86..6e4ec55e6e 100644 --- a/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/Logging.kt +++ b/save-agent/src/linuxX64Main/kotlin/com/saveourtool/save/agent/utils/Logging.kt @@ -9,10 +9,17 @@ package com.saveourtool.save.agent.utils import com.saveourtool.save.core.logging.logDebug import com.saveourtool.save.core.logging.logError import com.saveourtool.save.core.logging.logInfo +import io.ktor.client.plugins.logging.* import platform.linux.__NR_gettid import platform.posix.syscall +internal val ktorLogger = object : Logger { + override fun log(message: String) { + logInfoCustom("[HTTP Client] $message") + } +} + fun logErrorCustom(msg: String) = logError( "[tid ${syscall(__NR_gettid.toLong())}] $msg" ) diff --git a/save-agent/src/linuxX64Main/resources/agent.properties b/save-agent/src/linuxX64Main/resources/agent.properties index d4b0c1d4a1..093a7c7840 100644 --- a/save-agent/src/linuxX64Main/resources/agent.properties +++ b/save-agent/src/linuxX64Main/resources/agent.properties @@ -6,4 +6,5 @@ requestTimeoutMillis=60000 retry.attempts=5 retry.initialRetryMillis=2000 debug=true -cliCommand=echo Provide CLI command here \ No newline at end of file +cliCommand=echo Provide CLI command here +testSuitesDir=. diff --git a/save-agent/src/linuxX64Test/kotlin/com/saveourtool/save/agent/SaveAgentTest.kt b/save-agent/src/linuxX64Test/kotlin/com/saveourtool/save/agent/SaveAgentTest.kt index dc86e8e287..77eb41b099 100644 --- a/save-agent/src/linuxX64Test/kotlin/com/saveourtool/save/agent/SaveAgentTest.kt +++ b/save-agent/src/linuxX64Test/kotlin/com/saveourtool/save/agent/SaveAgentTest.kt @@ -6,13 +6,12 @@ import generated.SAVE_CORE_VERSION import io.ktor.client.HttpClient import io.ktor.client.engine.mock.MockEngine import io.ktor.client.engine.mock.respond -import io.ktor.client.plugins.json.JsonPlugin -import io.ktor.client.plugins.kotlinx.serializer.KotlinxSerializer +import io.ktor.client.plugins.contentnegotiation.* import io.ktor.http.ContentType import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.headersOf -import platform.posix.system +import io.ktor.serialization.kotlinx.json.* import kotlin.test.AfterTest import kotlin.test.BeforeTest @@ -32,8 +31,8 @@ class SaveAgentTest { if (Platform.osFamily == OsFamily.WINDOWS) it.copy(cliCommand = "save-$SAVE_CORE_VERSION-linuxX64.bat") else it } private val saveAgentForTest = SaveAgent(configuration, httpClient = HttpClient(MockEngine) { - install(JsonPlugin) { - serializer = KotlinxSerializer(json) + install(ContentNegotiation) { + json(json) } engine { addHandler { request -> @@ -75,7 +74,7 @@ class SaveAgentTest { @Test fun `should change state to FINISHED after SAVE CLI returns`() = runBlocking { - assertEquals(AgentState.STARTING, saveAgentForTest.state.value) + assertEquals(AgentState.BUSY, saveAgentForTest.state.value) runBlocking { saveAgentForTest.run { startSaveProcess("") } } assertEquals(AgentState.FINISHED, saveAgentForTest.state.value) } diff --git a/save-backend/backend-api-docs.json b/save-backend/backend-api-docs.json index 983914517e..14f0560af1 100644 --- a/save-backend/backend-api-docs.json +++ b/save-backend/backend-api-docs.json @@ -2029,7 +2029,7 @@ "/api/v1/image/upload": { "post": { "tags": [ - "download-files-controller" + "files" ], "operationId": "uploadImage", "parameters": [ @@ -2073,8 +2073,8 @@ } }, "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "*/*": { "schema": { @@ -2083,13 +2083,18 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } }, "/api/v1/files/{organizationName}/{projectName}/upload": { "post": { "tags": [ - "download-files-controller" + "files" ], "operationId": "upload", "parameters": [ @@ -2138,8 +2143,8 @@ } }, "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "*/*": { "schema": { @@ -2148,13 +2153,18 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } }, "/api/v1/files/{organizationName}/{projectName}/download": { "post": { "tags": [ - "download-files-controller" + "files" ], "operationId": "download", "parameters": [ @@ -2186,8 +2196,8 @@ "required": true }, "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "application/octet-stream": { "schema": { @@ -2230,13 +2240,18 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } }, "/api/v1/files/get-execution-info": { "post": { "tags": [ - "download-files-controller" + "files" ], "operationId": "getExecutionInfo", "requestBody": { @@ -2250,8 +2265,8 @@ "required": true }, "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "*/*": { "schema": { @@ -2294,13 +2309,18 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } }, "/api/v1/files/get-debug-info": { "post": { "tags": [ - "download-files-controller" + "files" ], "operationId": "getDebugInfo", "requestBody": { @@ -2314,8 +2334,8 @@ "required": true }, "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "*/*": { "schema": { @@ -2358,7 +2378,12 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } }, "/api/v1/executionRequestStandardTests": { @@ -4862,7 +4887,7 @@ "/api/v1/files/{organizationName}/{projectName}/list": { "get": { "tags": [ - "download-files-controller" + "files" ], "operationId": "list_1", "parameters": [ @@ -4884,8 +4909,8 @@ } ], "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "*/*": { "schema": { @@ -4897,7 +4922,12 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } }, "/api/v1/executionDto": { @@ -5359,6 +5389,246 @@ ] } }, + "/api/v1/contests/{contestName}/my-results": { + "get": { + "tags": [ + "contests" + ], + "summary": "Get your best results in contest.", + "description": "Get list of best results of your projects in a given contest.", + "operationId": "getBestResultsInUserProjects", + "parameters": [ + { + "name": "contestName", + "in": "path", + "description": "name of a contest", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-Authorization-Source", + "in": "header", + "required": true, + "example": "basic" + } + ], + "responses": { + "200": { + "description": "Successfully fetched your best results.", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ContestResult" + } + } + } + } + }, + "404": { + "description": "Either given project or given contest was not found.", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ContestResult" + } + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ContestResult" + } + } + } + } + } + }, + "security": [ + { + "basic": [] + } + ] + } + }, + "/api/v1/contests/{contestName}/executions/{organizationName}/{projectName}": { + "get": { + "tags": [ + "contests" + ], + "summary": "Get project executions in contest.", + "description": "Get list of execution of a project with given name in contest with given name.", + "operationId": "getContestExecutionsForProject", + "parameters": [ + { + "name": "contestName", + "in": "path", + "description": "name of an organization", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "organizationName", + "in": "path", + "description": "name of an organization", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "projectName", + "in": "path", + "description": "name of a project", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "404": { + "description": "Either contest is not found or project is not found or execution is not found.", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExecutionDto" + } + } + } + } + }, + "200": { + "description": "Successfully fetched latest project execution in contest.", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExecutionDto" + } + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "*/*": { + "schema": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ExecutionDto" + } + } + } + } + } + }, + "security": [ + { + "basic": [] + } + ] + } + }, + "/api/v1/contests/{contestName}/executions/{organizationName}/{projectName}/latest": { + "get": { + "tags": [ + "contests" + ], + "summary": "Get latest project execution in contest.", + "description": "Get latest execution of a project with given name in contest with given name.", + "operationId": "getLatestExecutionOfProjectInContest", + "parameters": [ + { + "name": "organizationName", + "in": "path", + "description": "name of an organization", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "projectName", + "in": "path", + "description": "name of a project", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "contestName", + "in": "path", + "description": "name of an organization", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "X-Authorization-Source", + "in": "header", + "required": true, + "example": "basic" + } + ], + "responses": { + "404": { + "description": "Either contest is not found or project is not found or execution is not found.", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/ExecutionDto" + } + } + } + }, + "200": { + "description": "Successfully fetched latest project execution in contest.", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/ExecutionDto" + } + } + } + }, + "401": { + "description": "Unauthorized", + "content": { + "*/*": { + "schema": { + "$ref": "#/components/schemas/ExecutionDto" + } + } + } + } + }, + "security": [ + { + "basic": [] + } + ] + } + }, "/api/v1/contests/{contestName}/enroll": { "get": { "tags": [ @@ -6355,7 +6625,7 @@ "/api/v1/files/{organizationName}/{projectName}/{creationTimestamp}": { "delete": { "tags": [ - "download-files-controller" + "files" ], "operationId": "delete", "parameters": [ @@ -6385,8 +6655,8 @@ } ], "responses": { - "200": { - "description": "OK", + "401": { + "description": "Unauthorized", "content": { "*/*": { "schema": { @@ -6395,7 +6665,12 @@ } } } - } + }, + "security": [ + { + "basic": [] + } + ] } } }, @@ -7184,6 +7459,7 @@ "type": { "type": "string", "enum": [ + "CONTEST", "GIT", "STANDARD" ] @@ -7361,6 +7637,7 @@ "type": { "type": "string", "enum": [ + "CONTEST", "GIT", "STANDARD" ] @@ -7417,8 +7694,11 @@ "ContestResult": { "required": [ "contestName", + "hasFailedTest", "organizationName", - "projectName" + "projectName", + "sdk", + "submissionStatus" ], "type": "object", "properties": { @@ -7434,6 +7714,25 @@ "score": { "type": "number", "format": "double" + }, + "submissionTime": { + "type": "string", + "format": "date-time" + }, + "submissionStatus": { + "type": "string", + "enum": [ + "ERROR", + "FINISHED", + "PENDING", + "RUNNING" + ] + }, + "sdk": { + "type": "string" + }, + "hasFailedTest": { + "type": "boolean" } } }, diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/ContestController.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/ContestController.kt index 0ff896b7db..aa91045e4e 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/ContestController.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/ContestController.kt @@ -256,7 +256,7 @@ internal class ContestController( contestService.createContestIfNotPresent(it) } .switchIfEmptyToResponseException(HttpStatus.CONFLICT) { - "Contest with name ${contestDto.name} is already present" + "Contest with name [${contestDto.name}] already exists!" } .map { ResponseEntity.ok("Contest has been successfully created!") diff --git a/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/DownloadFilesController.kt b/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/DownloadFilesController.kt index d753ec4872..116b28ee31 100644 --- a/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/DownloadFilesController.kt +++ b/save-backend/src/main/kotlin/com/saveourtool/save/backend/controllers/DownloadFilesController.kt @@ -2,16 +2,27 @@ package com.saveourtool.save.backend.controllers import com.saveourtool.save.agent.TestExecutionDto import com.saveourtool.save.backend.ByteBufferFluxResponse +import com.saveourtool.save.backend.configs.ApiSwaggerSupport import com.saveourtool.save.backend.repository.AgentRepository +import com.saveourtool.save.backend.service.ExecutionService import com.saveourtool.save.backend.service.OrganizationService import com.saveourtool.save.backend.service.ProjectService import com.saveourtool.save.backend.service.UserDetailsService import com.saveourtool.save.backend.storage.* +import com.saveourtool.save.backend.utils.blockingToMono import com.saveourtool.save.domain.* import com.saveourtool.save.from import com.saveourtool.save.permission.Permission import com.saveourtool.save.utils.AvatarType +import com.saveourtool.save.utils.switchIfEmptyToNotFound import com.saveourtool.save.v1 +import io.swagger.v3.oas.annotations.Operation +import io.swagger.v3.oas.annotations.Parameter +import io.swagger.v3.oas.annotations.Parameters +import io.swagger.v3.oas.annotations.enums.ParameterIn +import io.swagger.v3.oas.annotations.responses.ApiResponse +import io.swagger.v3.oas.annotations.tags.Tag +import io.swagger.v3.oas.annotations.tags.Tags import org.slf4j.LoggerFactory import org.springframework.http.HttpStatus @@ -32,6 +43,10 @@ import java.nio.ByteBuffer * A Spring controller for file downloading */ @RestController +@ApiSwaggerSupport +@Tags( + Tag(name = "files"), +) @Suppress("LongParameterList") class DownloadFilesController( private val fileStorage: FileStorage, @@ -39,6 +54,7 @@ class DownloadFilesController( private val debugInfoStorage: DebugInfoStorage, private val executionInfoStorage: ExecutionInfoStorage, private val agentRepository: AgentRepository, + private val executionService: ExecutionService, private val organizationService: OrganizationService, private val userDetailsService: UserDetailsService, private val projectService: ProjectService, @@ -105,6 +121,28 @@ class DownloadFilesController( @PathVariable projectName: String, ): Mono = downloadByFileKey(fileInfo.toStorageKey(), organizationName, projectName) + @PostMapping(path = ["/internal/files/download"], produces = [MediaType.APPLICATION_OCTET_STREAM_VALUE]) + @Operation( + method = "POST", + summary = "Download a file by execution ID and FileKey.", + description = "Download a file by execution ID and FileKey.", + ) + @Parameters( + Parameter(name = "executionId", `in` = ParameterIn.QUERY, description = "ID of an execution", required = true) + ) + @ApiResponse(responseCode = "200", description = "Returns content of the file.") + @ApiResponse(responseCode = "404", description = "Execution with provided ID is not found.") + fun downloadByExecutionId( + @RequestBody fileKey: FileKey, + @RequestParam executionId: Long, + ): Mono = blockingToMono { + executionService.findExecution(executionId) + } + .switchIfEmptyToNotFound() + .flatMap { execution -> + downloadByFileKey(fileKey, execution.project.organization.name, execution.project.name) + } + /** * @param fileKey a key [FileKey] of requested file * @param organizationName diff --git a/save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt b/save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt index fead50d0e4..41a756426f 100644 --- a/save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt +++ b/save-backend/src/test/kotlin/com/saveourtool/save/backend/DownloadFilesTest.kt @@ -6,6 +6,7 @@ import com.saveourtool.save.backend.configs.WebConfig import com.saveourtool.save.backend.controllers.DownloadFilesController import com.saveourtool.save.backend.repository.* import com.saveourtool.save.backend.security.ProjectPermissionEvaluator +import com.saveourtool.save.backend.service.ExecutionService import com.saveourtool.save.backend.service.OrganizationService import com.saveourtool.save.backend.service.ProjectService import com.saveourtool.save.backend.service.UserDetailsService @@ -79,6 +80,7 @@ import kotlin.io.path.* @MockBeans( MockBean(OrganizationService::class), MockBean(UserDetailsService::class), + MockBean(ExecutionService::class), ) class DownloadFilesTest { private val organization = Organization("Example.com", OrganizationStatus.CREATED, 1, null).apply { id = 2 } diff --git a/save-cloud-charts/save-cloud/README.md b/save-cloud-charts/save-cloud/README.md index 50a18fb904..17f54d7cd3 100644 --- a/save-cloud-charts/save-cloud/README.md +++ b/save-cloud-charts/save-cloud/README.md @@ -31,10 +31,7 @@ $ helm install save-cloud save-cloud-0.1.0.tgz --namespace save-cloud * Environment should be prepared: ```bash minikube ssh - docker@minikube:~$ for d in {repos, volumes, resources}; do sudo mkdir -p /tmp/save/$d && sudo chown -R 1000:100 /tmp/save/$d; done - docker@minikube:~$ sudo vi /lib/systemd/system/docker.service # change ExecSTart to allow HTTP connection to Docker daemon - docker@minikube:~$ sudo systemctl daemon-reload - docker@minikube:~$ sudo systemctl restart docker + docker@minikube:~$ for d in repos volumes resources; do sudo mkdir -p /tmp/save/$d && sudo chown -R 1000:100 /tmp/save/$d; done ``` * Install Helm chart using `values-minikube.yaml`: ```bash diff --git a/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml b/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml index 1649d1c183..2b4a04ff52 100644 --- a/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml +++ b/save-cloud-charts/save-cloud/templates/orchestrator-configmap.yaml @@ -9,7 +9,7 @@ data: orchestrator.kubernetes.apiServerUrl=http://kubernetes.default.svc orchestrator.kubernetes.serviceAccount=${POD_SERVICE_ACCOUNT} orchestrator.kubernetes.namespace=${POD_NAMESPACE} - orchestrator.kubernetes.pvcMountPath=/home/save-agent + orchestrator.kubernetes.pvcMountPath=/home/save-agent/resources server.shutdown=graceful management.endpoints.web.exposure.include=* diff --git a/save-cloud-charts/save-cloud/values-minikube.yaml b/save-cloud-charts/save-cloud/values-minikube.yaml index a93a3e63a2..a1cf1092cf 100644 --- a/save-cloud-charts/save-cloud/values-minikube.yaml +++ b/save-cloud-charts/save-cloud/values-minikube.yaml @@ -19,7 +19,9 @@ orchestrator: orchestrator.kubernetes.pvcAnnotations= orchestrator.kubernetes.pvcSize=5Gi orchestrator.kubernetes.pvcStorageSpec=hostPath:\n\ - \ path: /tmp/save/volumes + \ path: /tmp/save/volumes\n\ + \accessModes:\n\ + \- ReadWriteMany mysql: external: false ip: nil diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/kafka/KafkaMsg.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/kafka/KafkaMsg.kt new file mode 100644 index 0000000000..3c8ecd981a --- /dev/null +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/kafka/KafkaMsg.kt @@ -0,0 +1,11 @@ +package com.saveourtool.save.kafka + +/** + * kafka message + */ +interface KafkaMsg { + /** + * @property messageId + */ + val messageId: String? +} diff --git a/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/kafka/TestExecutionTaskDto.kt b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/kafka/TestExecutionTaskDto.kt new file mode 100644 index 0000000000..03fe23b5bf --- /dev/null +++ b/save-cloud-common/src/commonMain/kotlin/com/saveourtool/save/kafka/TestExecutionTaskDto.kt @@ -0,0 +1,14 @@ +package com.saveourtool.save.kafka + +import com.saveourtool.save.test.TestDto + +/** + * @property tests + * @property cliArgs + * @property messageId + */ +data class TestExecutionTaskDto( + val tests: List, + val cliArgs: String, + override val messageId: String? = null +) : KafkaMsg diff --git a/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/DateTimeUtils.kt b/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/DateTimeUtils.kt new file mode 100644 index 0000000000..c08d6291c5 --- /dev/null +++ b/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/DateTimeUtils.kt @@ -0,0 +1,16 @@ +/** + * Utility methods related to a Date and Time + */ + +package com.saveourtool.save.utils + +import kotlinx.datetime.Clock +import kotlinx.datetime.TimeZone +import kotlinx.datetime.toLocalDateTime + +actual typealias LocalDateTime = kotlinx.datetime.LocalDateTime + +/** + * @return current local date-time in UTC timezone + */ +fun getCurrentLocalDateTime() = Clock.System.now().toLocalDateTime(TimeZone.UTC) diff --git a/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/EntitiesUtils.kt b/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/EntitiesUtils.kt index 8651ac2fed..292a4155f9 100644 --- a/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/EntitiesUtils.kt +++ b/save-cloud-common/src/jsMain/kotlin/com/saveourtool/save/utils/EntitiesUtils.kt @@ -2,8 +2,6 @@ package com.saveourtool.save.utils -actual typealias LocalDateTime = kotlinx.datetime.LocalDateTime - actual enum class EnumType { /** Persist enumerated type property or field as an integer. */ ORDINAL, diff --git a/save-deploy/base-images/Dockerfile b/save-deploy/base-images/Dockerfile index bf6d24b86c..2ce71c75a8 100644 --- a/save-deploy/base-images/Dockerfile +++ b/save-deploy/base-images/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_IMAGE_NAME ARG BASE_IMAGE_TAG FROM $BASE_IMAGE_NAME:$BASE_IMAGE_TAG -RUN apt-get update && env DEBIAN_FRONTEND="noninteractive" apt-get install -y libcurl4-openssl-dev tzdata +RUN apt-get update && env DEBIAN_FRONTEND="noninteractive" apt-get install -y libcurl4-openssl-dev tzdata unzip RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime RUN rm -rf /var/lib/apt/lists/* @@ -13,5 +13,10 @@ RUN if [ "$BASE_IMAGE_NAME" = "python" ]; then \ ln -s $(which java) /usr/bin/java; \ fi -RUN groupadd --gid 1100 save-agent && useradd --uid 1100 --gid 1100 --create-home --shell /bin/sh save-agent +RUN groupadd --gid 1100 save-agent && \ + useradd --uid 1100 --gid 1100 --create-home --shell /bin/sh save-agent && \ + # `WORKDIR` directive creates the directory as `root` user unless the directory already exists + mkdir /home/save-agent/save-execution && \ + chown -R 1100:1100 /home/save-agent/save-execution +USER save-agent WORKDIR /home/save-agent/save-execution \ No newline at end of file diff --git a/save-frontend/build.gradle.kts b/save-frontend/build.gradle.kts index 215844483d..a01cd4ba55 100644 --- a/save-frontend/build.gradle.kts +++ b/save-frontend/build.gradle.kts @@ -76,8 +76,6 @@ kotlin { implementation(npm("react-tsparticles", "1.42.1")) implementation(npm("tsparticles", "2.1.3")) implementation(npm("jquery", "3.6.0")) - // can be very useful for CSS animation: - // implementation(npm("animate.css", "4.1.1")) // BS5: implementation(npm("@popperjs/core", "2.11.0")) implementation(npm("popper.js", "1.16.1")) // BS5: implementation(npm("bootstrap", "5.0.1")) diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/App.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/App.kt index 87cd873419..b819b1b381 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/App.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/App.kt @@ -11,10 +11,11 @@ import com.saveourtool.save.execution.TestExecutionFilters import com.saveourtool.save.frontend.components.* import com.saveourtool.save.frontend.components.basic.scrollToTopButton import com.saveourtool.save.frontend.components.views.* -import com.saveourtool.save.frontend.components.views.usersettingsview.UserSettingsEmailMenuView -import com.saveourtool.save.frontend.components.views.usersettingsview.UserSettingsOrganizationsMenuView -import com.saveourtool.save.frontend.components.views.usersettingsview.UserSettingsProfileMenuView -import com.saveourtool.save.frontend.components.views.usersettingsview.UserSettingsTokenMenuView +import com.saveourtool.save.frontend.components.views.contests.ContestListView +import com.saveourtool.save.frontend.components.views.usersettings.UserSettingsEmailMenuView +import com.saveourtool.save.frontend.components.views.usersettings.UserSettingsOrganizationsMenuView +import com.saveourtool.save.frontend.components.views.usersettings.UserSettingsProfileMenuView +import com.saveourtool.save.frontend.components.views.usersettings.UserSettingsTokenMenuView import com.saveourtool.save.frontend.components.views.welcome.WelcomeView import com.saveourtool.save.frontend.externals.modal.ReactModal import com.saveourtool.save.frontend.http.getUser diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/ScoreCard.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/ScoreCard.kt index 1f32b01a7f..49b82e955f 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/ScoreCard.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/ScoreCard.kt @@ -9,7 +9,7 @@ import react.dom.aria.AriaRole import react.dom.aria.ariaValueMax import react.dom.aria.ariaValueMin import react.dom.aria.ariaValueNow -import react.dom.html.ReactHTML +import react.dom.html.ReactHTML.a import react.dom.html.ReactHTML.div import react.dom.html.ReactHTML.h6 @@ -95,7 +95,7 @@ private fun scoreCard() = FC { props -> } props.url?.let { - ReactHTML.a { + a { href = props.url +props.name } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/UserBoard.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/UserBoard.kt index c5fda53ede..2d29e5d940 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/UserBoard.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/UserBoard.kt @@ -58,7 +58,7 @@ private fun userBoard() = FC { props -> "/api/$v1/avatar$path" } ?: run { - "img/user.svg" + "img/undraw_profile.svg" } alt = "" } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/organizations/OrganizationContestsMenu.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/organizations/OrganizationContestsMenu.kt index 242d3914ed..17817fdabd 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/organizations/OrganizationContestsMenu.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/basic/organizations/OrganizationContestsMenu.kt @@ -19,9 +19,10 @@ import org.w3c.fetch.Response import react.* import react.dom.html.ButtonType -import react.dom.html.ReactHTML +import react.dom.html.ReactHTML.a import react.dom.html.ReactHTML.button import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.td import react.table.columns /** @@ -34,8 +35,8 @@ private val contestsTable: FC> = tabl columns = columns { column(id = "name", header = "Contest Name", { name }) { cellProps -> Fragment.create { - ReactHTML.td { - ReactHTML.a { + td { + a { href = "#/contests/${cellProps.row.original.name}" +cellProps.value } @@ -44,21 +45,21 @@ private val contestsTable: FC> = tabl } column(id = "description", header = "Description", { description }) { cellProps -> Fragment.create { - ReactHTML.td { + td { +(cellProps.value ?: "Description is not provided") } } } column(id = "start_time", header = "Start Time", { startTime.toString() }) { cellProps -> Fragment.create { - ReactHTML.td { + td { +cellProps.value.replace("T", " ") } } } column(id = "end_time", header = "End Time", { endTime.toString() }) { cellProps -> Fragment.create { - ReactHTML.td { + td { +cellProps.value.replace("T", " ") } } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/AwesomeBenchmarksView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/AwesomeBenchmarksView.kt index 84da0c18a5..068a87a5a8 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/AwesomeBenchmarksView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/AwesomeBenchmarksView.kt @@ -87,7 +87,13 @@ class AwesomeBenchmarksView : AbstractView { + style = jso { cursor = "pointer".unsafeCast() } @@ -486,22 +492,19 @@ class AwesomeBenchmarksView : AbstractView = get( - "$apiUrl/${FrontendRoutes.AWESOME_BENCHMARKS.path}", - headers, - loadingHandler = ::classLoadingHandler, - ).decodeFromJsonString() - - setState { - benchmarks = response - } + val response: List = get( + "$apiUrl/${FrontendRoutes.AWESOME_BENCHMARKS.path}", + headers, + loadingHandler = ::classLoadingHandler, + ).decodeFromJsonString() + + setState { + benchmarks = response } } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/CollectionView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/CollectionView.kt index 72bda8451e..25a3a6ca0a 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/CollectionView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/CollectionView.kt @@ -92,21 +92,19 @@ class CollectionView : AbstractView(false) { override fun ChildrenBuilder.render() { div { hidden = (props.currentUserInfo == null) - button { - type = ButtonType.button - className = ClassName("btn btn-primary mb-2 mr-2") - a { - className = ClassName("text-light") - href = "#/${FrontendRoutes.CREATE_PROJECT.path}/" + a { + href = "#/${FrontendRoutes.CREATE_PROJECT.path}/" + button { + type = ButtonType.button + className = ClassName("btn btn-outline-primary mb-2 mr-2") +"Add new tested tool" } } - button { - type = ButtonType.button - className = ClassName("btn btn-primary mb-2") - a { - className = ClassName("text-light") - href = "#/${FrontendRoutes.CREATE_ORGANIZATION.path}/" + a { + href = "#/${FrontendRoutes.CREATE_ORGANIZATION.path}/" + button { + type = ButtonType.button + className = ClassName("btn btn-outline-primary mb-2") +"Add new organization" } } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/ContestListView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/ContestListView.kt deleted file mode 100644 index 3ad95ea4e3..0000000000 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/ContestListView.kt +++ /dev/null @@ -1,177 +0,0 @@ -@file:Suppress("FILE_WILDCARD_IMPORTS", "WildcardImport", "HEADER_MISSING_IN_NON_SINGLE_CLASS_FILE") - -package com.saveourtool.save.frontend.components.views - -import com.saveourtool.save.entities.ContestDto -import com.saveourtool.save.frontend.components.RequestStatusContext -import com.saveourtool.save.frontend.components.basic.ContestNameProps -import com.saveourtool.save.frontend.components.basic.showContestEnrollerModal -import com.saveourtool.save.frontend.components.requestStatusContext -import com.saveourtool.save.frontend.components.tables.tableComponent -import com.saveourtool.save.frontend.utils.* -import com.saveourtool.save.frontend.utils.classLoadingHandler -import com.saveourtool.save.info.UserInfo -import com.saveourtool.save.validation.FrontendRoutes - -import org.w3c.fetch.Headers -import react.* -import react.dom.html.ReactHTML.a -import react.dom.html.ReactHTML.td -import react.table.columns - -/** - * [Props] retrieved from router - */ -@Suppress("MISSING_KDOC_CLASS_ELEMENTS") -external interface ContestListViewProps : Props { - var currentUserInfo: UserInfo? -} - -/** - * [State] of [ContestListView] component - */ -external interface ContestListViewState : State { - /** - * Flag to show project selector modal - */ - var isProjectSelectorModalOpen: Boolean - - /** - * Flag th show confirmation modal - */ - var isConfirmationWindowOpen: Boolean - - /** - * Name of a contest selected for enrollment - */ - var selectedContestName: String? - - /** - * Enrollment response received from backend - */ - var enrollmentResponse: String? -} - -/** - * A view with collection of contests - */ -@JsExport -@OptIn(ExperimentalJsExport::class) -class ContestListView : AbstractView(false) { - private val openParticipateModal: (String) -> Unit = { contestName -> - setState { - selectedContestName = contestName - isProjectSelectorModalOpen = true - } - } - - @Suppress("MAGIC_NUMBER") - private val contestsTable = tableComponent( - columns = columns { - column(id = "name", header = "Contest Name", { this }) { cellProps -> - Fragment.create { - td { - onClick = { - openParticipateModal(cellProps.value.name) - } - a { - href = "#/${FrontendRoutes.CONTESTS.path}/${cellProps.row.original.name}" - +cellProps.value.name - } - } - } - } - column(id = "description", header = "Description", { this }) { cellProps -> - Fragment.create { - td { - onClick = { - openParticipateModal(cellProps.value.name) - } - +(cellProps.value.description ?: "Description is not provided") - } - } - } - column(id = "start_time", header = "Start Time", { this }) { cellProps -> - Fragment.create { - td { - onClick = { - openParticipateModal(cellProps.value.name) - } - +cellProps.value.startTime.toString().replace("T", " ") - } - } - } - column(id = "end_time", header = "End Time", { this }) { cellProps -> - Fragment.create { - td { - onClick = { - openParticipateModal(cellProps.value.name) - } - +cellProps.value.endTime.toString().replace("T", " ") - } - } - } - }, - initialPageSize = 10, - useServerPaging = false, - usePageSelection = false, - ) - init { - state.selectedContestName = null - state.isProjectSelectorModalOpen = false - state.enrollmentResponse = null - state.isConfirmationWindowOpen = false - } - @Suppress( - "EMPTY_BLOCK_STRUCTURE_ERROR", - "TOO_LONG_FUNCTION", - "MAGIC_NUMBER", - "LongMethod", - ) - override fun ChildrenBuilder.render() { - showContestEnrollerModal( - state.isProjectSelectorModalOpen, - ContestNameProps(state.selectedContestName ?: ""), - { setState { isProjectSelectorModalOpen = false } } - ) { - setState { - enrollmentResponse = it - isConfirmationWindowOpen = true - isProjectSelectorModalOpen = false - } - } - runErrorModal( - state.isConfirmationWindowOpen, - "Contest Registration", - state.enrollmentResponse ?: "", - "Ok" - ) { - setState { isConfirmationWindowOpen = false } - } - contestsTable { - getData = { _, _ -> - val response = get( - url = "$apiUrl/contests/active", - headers = Headers().also { - it.set("Accept", "application/json") - }, - loadingHandler = ::classLoadingHandler, - ) - if (response.ok) { - response.unsafeMap { - it.decodeFromJsonString>() - } - .toTypedArray() - } else { - emptyArray() - } - } - } - } - - companion object : RStatics>(ContestListView::class) { - init { - contextType = requestStatusContext - } - } -} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/OrganizationView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/OrganizationView.kt index 722faf1edd..eb4466f9ec 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/OrganizationView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/OrganizationView.kt @@ -689,13 +689,12 @@ class OrganizationView : AbstractView( } if (state.selfRole.isHigherOrEqualThan(Role.ADMIN)) { - button { - type = ButtonType.button - className = ClassName("btn btn-primary") - a { - className = ClassName("text-light") - href = "#/${FrontendRoutes.CREATE_PROJECT.path}/" - +"+ New Tool" + a { + href = "#/${FrontendRoutes.CREATE_PROJECT.path}/" + button { + type = ButtonType.button + className = ClassName("btn btn-outline-info") + +"Add Tool" } } } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ContestListCard.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ContestListCard.kt new file mode 100644 index 0000000000..c6fa60a078 --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ContestListCard.kt @@ -0,0 +1,151 @@ +/** + * a card with contests list from ContestListView + */ + +package com.saveourtool.save.frontend.components.views.contests + +import com.saveourtool.save.entities.ContestDto +import com.saveourtool.save.frontend.externals.fontawesome.faArrowRight +import com.saveourtool.save.frontend.externals.fontawesome.faCode +import com.saveourtool.save.frontend.externals.fontawesome.fontAwesomeIcon +import com.saveourtool.save.validation.FrontendRoutes + +import csstype.ClassName +import csstype.rem +import react.ChildrenBuilder +import react.FC +import react.Props +import react.dom.html.ButtonType +import react.dom.html.ReactHTML.a +import react.dom.html.ReactHTML.button +import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.img +import react.dom.html.ReactHTML.p +import react.dom.html.ReactHTML.strong + +import kotlinx.js.jso + +val contestListFc = contestList() + +/** + * this enum is used in a tab for contests + */ +enum class ContestTypesTab { + ACTIVE, FINISHED, PLANNED +} + +/** + * properties for contestList FC + */ +external interface ContestListProps : Props { + /** + * contest tab selected by user (propagated from state) + */ + var selectedTab: String? + + /** + * callback to update state with a tab selection + */ + var updateTabState: (String) -> Unit + + /** + * all finished contests + */ + var finishedContests: Set + + /** + * all active contests + */ + var activeContests: Set + + /** + * callback to set selected contest (will trigger a modal window with a participation modal window) + */ + var updateSelectedContestName: (String) -> Unit +} + +private fun ChildrenBuilder.contests(props: ContestListProps) { + when (props.selectedTab) { + ContestTypesTab.ACTIVE.name -> contestListTable(props.activeContests, props.updateSelectedContestName) + ContestTypesTab.FINISHED.name -> contestListTable(props.finishedContests, props.updateSelectedContestName) + ContestTypesTab.PLANNED.name -> { + // FixMe: Add planned contests + } + } +} + +@Suppress("MAGIC_NUMBER") +private fun ChildrenBuilder.contestListTable(contests: Set, updateSelectedContestName: (String) -> Unit) { + contests.forEachIndexed { i, contest -> + div { + className = ClassName("media text-muted pb-3") + img { + className = ClassName("rounded") + asDynamic()["data-src"] = + "holder.js/32x32?theme=thumb&bg=007bff&fg=007bff&size=1" + src = "img/undraw_code_inspection_bdl7.svg" + asDynamic()["data-holder-rendered"] = "true" + style = jso { + width = 4.2.rem + } + } + + p { + className = ClassName("media-body pb-3 mb-0 small lh-125 border-bottom border-gray text-left") + strong { + className = ClassName("d-block text-gray-dark") + +contest.name + } + +(contest.description ?: "") + + div { + className = ClassName("navbar-landing mt-3") + button { + type = ButtonType.button + className = ClassName("btn btn-outline-success ml-auto mr-2") + onClick = { + // FixMe: fix the selector here + updateSelectedContestName(contest.name) + } + +"Enroll" + } + a { + className = ClassName("btn btn-outline-info mr-2") + href = "#/${FrontendRoutes.CONTESTS.path}/${contest.name}" + +"Rules and more " + fontAwesomeIcon(icon = faArrowRight) + } + } + } + } + } +} + +/** + * @return functional component that render the stylish table with contests + */ +fun contestList() = FC { props -> + div { + className = ClassName("col-lg-9") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + minHeight = 30.rem + } + + div { + className = ClassName("col") + + title(" Available Contests", faCode) + + tab( + props.selectedTab, + ContestTypesTab.values().map { it.name }, + props.updateTabState + ) + + contests(props) + } + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ContestListView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ContestListView.kt new file mode 100644 index 0000000000..7bfcd6115b --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ContestListView.kt @@ -0,0 +1,243 @@ +/** + * Contests "market" - showcase for users, where they can navigate and check contests + */ + +@file:Suppress("FILE_WILDCARD_IMPORTS", "WildcardImport", "MAGIC_NUMBER") + +package com.saveourtool.save.frontend.components.views.contests + +import com.saveourtool.save.entities.ContestDto +import com.saveourtool.save.frontend.components.RequestStatusContext +import com.saveourtool.save.frontend.components.basic.ContestNameProps +import com.saveourtool.save.frontend.components.basic.showContestEnrollerModal +import com.saveourtool.save.frontend.components.requestStatusContext +import com.saveourtool.save.frontend.components.views.AbstractView +import com.saveourtool.save.frontend.utils.* +import com.saveourtool.save.frontend.utils.classLoadingHandler +import com.saveourtool.save.info.UserInfo +import com.saveourtool.save.utils.LocalDateTime +import com.saveourtool.save.utils.getCurrentLocalDateTime + +import csstype.ClassName +import csstype.rem +import org.w3c.fetch.Headers +import react.* +import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.main + +import kotlinx.coroutines.launch +import kotlinx.js.jso + +/** + * [Props] retrieved from router + */ +@Suppress("MISSING_KDOC_CLASS_ELEMENTS") +external interface ContestListViewProps : Props { + var currentUserInfo: UserInfo? +} + +/** + * [State] of [ContestListView] component + */ +external interface ContestListViewState : State { + /** + * list of contests that are not expired yet (deadline is not reached) + */ + var activeContests: Set + + /** + * list of contests that are expired (deadline has reached) + */ + var finishedContests: Set + + /** + * current time + */ + var currentDateTime: LocalDateTime + + /** + * selected tab + */ + var selectedContestsTab: String? + + /** + * selected tab + */ + var selectedRatingTab: String? + + /** + * Flag to show project selector modal + */ + var isProjectSelectorModalOpen: Boolean + + /** + * Flag th show confirmation modal + */ + var isConfirmationWindowOpen: Boolean + + /** + * Name of a contest selected for enrollment + */ + var selectedContestName: String? + + /** + * Enrollment response received from backend + */ + var enrollmentResponse: String? +} + +/** + * A view with collection of contests + */ +@JsExport +@OptIn(ExperimentalJsExport::class) +class ContestListView : AbstractView() { + private val openParticipateModal: (String) -> Unit = { contestName -> + setState { + selectedContestName = contestName + isProjectSelectorModalOpen = true + } + } + + init { + state.selectedRatingTab = UserRatingTab.ORGANIZATIONS.name + state.selectedContestsTab = ContestTypesTab.ACTIVE.name + state.finishedContests = emptySet() + state.activeContests = emptySet() + state.currentDateTime = getCurrentLocalDateTime() + } + + override fun componentDidMount() { + super.componentDidMount() + scope.launch { + getAndInitActiveContests() + getAndInitFinishedContests() + } + } + + @Suppress("TOO_LONG_FUNCTION", "LongMethod") + override fun ChildrenBuilder.render() { + showContestEnrollerModal( + state.isProjectSelectorModalOpen, + ContestNameProps(state.selectedContestName ?: ""), + { setState { isProjectSelectorModalOpen = false } } + ) { + setState { + enrollmentResponse = it + isConfirmationWindowOpen = true + isProjectSelectorModalOpen = false + } + } + runErrorModal( + state.isConfirmationWindowOpen, + "Contest Registration", + state.enrollmentResponse ?: "", + "Ok" + ) { + setState { isConfirmationWindowOpen = false } + } + + main { + className = ClassName("main-content mt-0 ps") + div { + className = ClassName("page-header align-items-start min-vh-100") + div { + className = ClassName("row justify-content-center") + div { + className = ClassName("col-lg-9") + div { + className = ClassName("row mb-2") + featuredContest() + newContestsCard() + } + + div { + className = ClassName("row mb-2") + div { + className = ClassName("col-lg-5") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + minHeight = 7.rem + } + } + } + div { + className = ClassName("col-lg-5") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + minHeight = 7.rem + } + } + } + + proposeContest() + } + + div { + className = ClassName("row mb-2") + userRatingFc { + selectedTab = state.selectedRatingTab + updateTabState = { setState { selectedRatingTab = it } } + } + + contestListFc { + activeContests = state.activeContests + finishedContests = state.finishedContests + selectedTab = state.selectedContestsTab + updateTabState = { setState { selectedContestsTab = it } } + updateSelectedContestName = { setState { selectedContestName = it } } + } + } + } + } + } + } + } + + private suspend fun getAndInitActiveContests() { + getAndInitContests("active") { + setState { + activeContests = it + } + } + } + + private suspend fun getAndInitFinishedContests() { + getAndInitContests("finished") { + setState { + finishedContests = it + } + } + } + + private suspend fun getAndInitContests(url: String, setState: (Set) -> Unit) { + val response = get( + url = "$apiUrl/contests/$url", + headers = Headers().also { + it.set("Accept", "application/json") + }, + loadingHandler = ::classLoadingHandler, + ) + val contestsUpdate = if (response.ok) { + response.unsafeMap { + it.decodeFromJsonString>() + } + .toTypedArray() + } else { + emptyArray() + }.toSet() + + setState(contestsUpdate) + } + + companion object : + RStatics>( + ContestListView::class + ) { + init { + contextType = requestStatusContext + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/CountDownTimer.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/CountDownTimer.kt new file mode 100644 index 0000000000..7cf0415d62 --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/CountDownTimer.kt @@ -0,0 +1,30 @@ +/** + * Countdown timer FC that is used to show how much time is left for this particular contest. + * The idea is to motivate user to participate in contests from this view. + */ + +@file:Suppress("diktat") + +package com.saveourtool.save.frontend.components.views.contests + +import com.saveourtool.save.entities.ContestDto +import react.FC +import react.Props +import kotlin.js.Date + +/** + * constant component + */ +val countDownFc = countDownTimer() + +external interface CountDownProps : Props { + var contests: Array +} + +/** + * @return + */ +fun countDownTimer() = FC {props -> + console.log(Date().toLocaleString()) + TODO("This will be finished in phaze 2: this countdown timer should be in a featured contest") +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/FeaturedContest.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/FeaturedContest.kt new file mode 100644 index 0000000000..ded8c14b4c --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/FeaturedContest.kt @@ -0,0 +1,94 @@ +/** + * A card with a FEATURED contest (left top card) + */ + +package com.saveourtool.save.frontend.components.views.contests + +import com.saveourtool.save.frontend.externals.fontawesome.faArrowRight +import com.saveourtool.save.frontend.externals.fontawesome.fontAwesomeIcon + +import csstype.ClassName +import csstype.rem +import react.ChildrenBuilder +import react.dom.html.ButtonType +import react.dom.html.ReactHTML.a +import react.dom.html.ReactHTML.button +import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.h3 +import react.dom.html.ReactHTML.img +import react.dom.html.ReactHTML.p +import react.dom.html.ReactHTML.strong + +import kotlinx.js.jso + +/** + * Rendering of featured contest card + */ +@Suppress("MAGIC_NUMBER") +fun ChildrenBuilder.featuredContest() { + div { + className = ClassName("col-lg-6") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + height = 14.rem + } + + image() + + div { + className = ClassName("card-body d-flex flex-column align-items-start") + strong { + className = ClassName("d-inline-block mb-2 text-info") + +"Featured Contest" + } + h3 { + className = ClassName("mb-0") + a { + className = ClassName("text-dark") + href = "#" + +"Contest NAME" + } + } + p { + className = ClassName("card-text mb-auto") + +"Contest DESCRIPTION SHORT" + } + div { + className = ClassName("row") + button { + type = ButtonType.button + className = ClassName("btn btn-sm btn-outline-primary mr-1") + onClick = { + // FixMe: add enrollment logic - modal window + } + +"Enroll" + } + + button { + type = ButtonType.button + className = ClassName("btn btn-sm btn-outline-success") + onClick = { + // FixMe: link or a modal window here? + } + +"Description " + fontAwesomeIcon(icon = faArrowRight) + } + } + } + } + } +} + +@Suppress("MAGIC_NUMBER") +private fun ChildrenBuilder.image() { + img { + className = ClassName("card-img-right flex-auto d-none d-md-block") + asDynamic()["data-src"] = "holder.js/200x250?theme=thumb" + src = "img/undraw_certificate_re_yadi.svg" + asDynamic()["data-holder-rendered"] = "true" + style = jso { + width = 12.rem + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/NewContestsCard.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/NewContestsCard.kt new file mode 100644 index 0000000000..a6b9fa58ca --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/NewContestsCard.kt @@ -0,0 +1,71 @@ +/** + * card with newly added contests + */ + +package com.saveourtool.save.frontend.components.views.contests + +import csstype.ClassName +import csstype.rem +import react.ChildrenBuilder +import react.dom.html.ReactHTML.a +import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.h3 +import react.dom.html.ReactHTML.img +import react.dom.html.ReactHTML.p +import react.dom.html.ReactHTML.strong + +import kotlinx.js.jso + +/** + * rendering of newly added contests + */ +@Suppress("MAGIC_NUMBER") +fun ChildrenBuilder.newContestsCard() { + div { + className = ClassName("col-lg-6") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + height = 14.rem + } + + div { + className = ClassName("card-body d-flex flex-column align-items-start") + strong { + className = ClassName("d-inline-block mb-2 text-success") + +"""New contests""" + } + h3 { + className = ClassName("mb-0") + a { + className = ClassName("text-dark") + href = "#" + +"Hurry up!" + } + } + p { + className = ClassName("card-text mb-auto") + +"Checkout and participate in newest contests!" + } + a { + href = "https://github.com/saveourtool/save-cloud" + +"Link " + } + a { + href = "https://github.com/saveourtool/save" + +" Other link" + } + } + + img { + className = ClassName("card-img-right flex-auto d-none d-md-block") + asDynamic()["data-src"] = "holder.js/200x250?theme=thumb" + src = "img/undraw_exciting_news_re_y1iw.svg" + asDynamic()["data-holder-rendered"] = "true" + style = jso { + width = 12.rem + } + } + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ProposeYourContest.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ProposeYourContest.kt new file mode 100644 index 0000000000..c44ee57466 --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/ProposeYourContest.kt @@ -0,0 +1,30 @@ +/** + * Card with our e-mail - where you can propose a contest + */ + +package com.saveourtool.save.frontend.components.views.contests + +import csstype.ClassName +import csstype.rem +import react.ChildrenBuilder +import react.dom.html.ReactHTML.div + +import kotlinx.js.jso + +/** + * rendering of a card where we suggest to propose new custom contests + */ +@Suppress("MAGIC_NUMBER") +fun ChildrenBuilder.proposeContest() { + div { + className = ClassName("col-lg-2") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + minHeight = 7.rem + } + + // FixMe: Want to propose contest? Write e-mail! undraw_mailbox_re_dvds.svg + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/UserRating.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/UserRating.kt new file mode 100644 index 0000000000..e60b1c10a9 --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/UserRating.kt @@ -0,0 +1,69 @@ +/** + * Card for the rendering of ratings: for organizations and tools + */ + +package com.saveourtool.save.frontend.components.views.contests + +import com.saveourtool.save.frontend.externals.fontawesome.faArrowRight +import com.saveourtool.save.frontend.externals.fontawesome.faTrophy +import com.saveourtool.save.frontend.externals.fontawesome.fontAwesomeIcon + +import csstype.* +import react.FC +import react.Props +import react.dom.html.ReactHTML.a +import react.dom.html.ReactHTML.div + +import kotlinx.js.jso + +val userRatingFc = userRating() + +/** + * Enum that contains values for the tab that is used in rating card + */ +enum class UserRatingTab { + ORGANIZATIONS, TOOLS +} + +/** + * properties for rating fc + */ +external interface UserRatingProps : Props { + /** + * string value of the selected tab: organization/tools/etc. + */ + var selectedTab: String? + + /** + * callback that will be passed into this fc from the view + */ + var updateTabState: (String) -> Unit +} + +/** + * @return functional component for the rating card + */ +fun userRating() = FC { props -> + div { + className = ClassName("col-lg-3") + div { + className = ClassName("card flex-md-row mb-1 box-shadow") + style = jso { + minHeight = 30.rem + } + + div { + className = ClassName("col") + title(" Global Rating", faTrophy) + tab(props.selectedTab, UserRatingTab.values().map { it.name }, props.updateTabState) + // FixMe: user rating here + a { + // FixMe: new view on this link + href = "" + +"View more " + fontAwesomeIcon(faArrowRight) + } + } + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/Utils.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/Utils.kt new file mode 100644 index 0000000000..2568863c0b --- /dev/null +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/contests/Utils.kt @@ -0,0 +1,86 @@ +/** + * Utility methods used for rendering of contest list view + */ + +package com.saveourtool.save.frontend.components.views.contests + +import com.saveourtool.save.frontend.externals.fontawesome.FontAwesomeIconModule +import com.saveourtool.save.frontend.externals.fontawesome.fontAwesomeIcon + +import csstype.* +import react.ChildrenBuilder +import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.h4 +import react.dom.html.ReactHTML.li +import react.dom.html.ReactHTML.nav +import react.dom.html.ReactHTML.p + +import kotlinx.js.jso + +/** + * @param title + * @param icon + */ +fun ChildrenBuilder.title(title: String, icon: FontAwesomeIconModule) { + div { + className = ClassName("row") + style = jso { + justifyContent = JustifyContent.center + display = Display.flex + } + h4 { + style = jso { + color = "#5a5c69".unsafeCast() + } + fontAwesomeIcon(icon = icon) + + className = ClassName("mt-2 mb-4") + +title + } + } +} + +/** + * @param selectedTab + * @param tabsList + * @param updateTabState + */ +fun ChildrenBuilder.tab(selectedTab: String?, tabsList: List, updateTabState: (String) -> Unit) { + div { + className = ClassName("row") + style = jso { + justifyContent = JustifyContent.center + display = Display.flex + } + + nav { + className = ClassName("nav nav-tabs mb-4") + tabsList.forEachIndexed { i, value -> + li { + className = ClassName("nav-item") + val classVal = + if ((i == 0 && selectedTab == null) || selectedTab == value) { + " active font-weight-bold" + } else { + "" + } + p { + className = ClassName("nav-link $classVal text-gray-800") + onClick = { + kotlinx.js.console.log(value) + kotlinx.js.console.log(selectedTab) + if (selectedTab != value) { + updateTabState(value) + } + } + style = jso { + cursor = "pointer".unsafeCast() + } + + +value + } + } + } + } + } +} diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsEmailMenuView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsEmailMenuView.kt similarity index 99% rename from save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsEmailMenuView.kt rename to save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsEmailMenuView.kt index 0a0acf5c50..e344ed2cb8 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsEmailMenuView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsEmailMenuView.kt @@ -1,4 +1,4 @@ -package com.saveourtool.save.frontend.components.views.usersettingsview +package com.saveourtool.save.frontend.components.views.usersettings import com.saveourtool.save.frontend.components.basic.InputTypes import com.saveourtool.save.frontend.components.basic.cardComponent diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsOrganizationsMenuView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsOrganizationsMenuView.kt similarity index 99% rename from save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsOrganizationsMenuView.kt rename to save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsOrganizationsMenuView.kt index cd20802269..7511db54a3 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsOrganizationsMenuView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsOrganizationsMenuView.kt @@ -1,4 +1,4 @@ -package com.saveourtool.save.frontend.components.views.usersettingsview +package com.saveourtool.save.frontend.components.views.usersettings import com.saveourtool.save.domain.Role import com.saveourtool.save.frontend.components.basic.cardComponent diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsProfileMenuView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsProfileMenuView.kt similarity index 99% rename from save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsProfileMenuView.kt rename to save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsProfileMenuView.kt index c14bb00fe9..fe4904bf21 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsProfileMenuView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsProfileMenuView.kt @@ -1,4 +1,4 @@ -package com.saveourtool.save.frontend.components.views.usersettingsview +package com.saveourtool.save.frontend.components.views.usersettings import com.saveourtool.save.frontend.components.basic.InputTypes import com.saveourtool.save.frontend.components.basic.cardComponent diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsTokenMenuView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsTokenMenuView.kt similarity index 99% rename from save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsTokenMenuView.kt rename to save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsTokenMenuView.kt index 50dba11ef5..e85288eed2 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsTokenMenuView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsTokenMenuView.kt @@ -1,4 +1,4 @@ -package com.saveourtool.save.frontend.components.views.usersettingsview +package com.saveourtool.save.frontend.components.views.usersettings import com.saveourtool.save.frontend.components.basic.cardComponent import com.saveourtool.save.frontend.utils.apiUrl @@ -9,7 +9,6 @@ import csstype.ClassName import kotlinext.js.assign import org.w3c.fetch.Headers import react.FC -import react.dom.* import react.dom.html.ButtonType import react.dom.html.ReactHTML.button import react.dom.html.ReactHTML.div diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsView.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsView.kt similarity index 99% rename from save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsView.kt rename to save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsView.kt index b2e3dfe259..0912b83985 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettingsview/UserSettingsView.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/usersettings/UserSettingsView.kt @@ -2,7 +2,7 @@ * A view with settings user */ -package com.saveourtool.save.frontend.components.views.usersettingsview +package com.saveourtool.save.frontend.components.views.usersettings import com.saveourtool.save.domain.ImageInfo import com.saveourtool.save.entities.OrganizationDto @@ -168,7 +168,7 @@ abstract class UserSettingsView : AbstractView(true) { className = ClassName("card-body") div { className = ClassName("text-sm") - menuTextAndLink("Contests", "/#/${FrontendRoutes.CONTESTS.path}", faBell) + menuTextAndLink("Contests", "/#/${FrontendRoutes.CONTESTS.path}", faCode) hrNoMargin() menuTextAndLink("List of Projects", "#/${FrontendRoutes.PROJECTS.path}", faExternalLinkAlt) hrNoMargin() @@ -349,7 +349,7 @@ class WelcomeView : AbstractView(true) { } private fun ChildrenBuilder.hrNoMargin() = - ReactHTML.hr { + hr { style = jso { marginTop = "0.0em".unsafeCast() marginBottom = "0.0em".unsafeCast() diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/AwesomeBenchmarks.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/AwesomeBenchmarks.kt index d2eaaf5b4e..914947b945 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/AwesomeBenchmarks.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/AwesomeBenchmarks.kt @@ -4,7 +4,7 @@ import com.saveourtool.save.frontend.externals.animations.* import csstype.Width import react.ChildrenBuilder -import react.dom.html.ReactHTML +import react.dom.html.ReactHTML.img import kotlinx.js.jso @@ -18,7 +18,7 @@ object AwesomeBenchmarks : WelcomePager { } private fun ChildrenBuilder.renderAnimatedPage() { - ReactHTML.img { + img { style = jso { width = "140%".unsafeCast() } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/BobPage.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/BobPage.kt index d44aa1f29c..0790df7d18 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/BobPage.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/BobPage.kt @@ -4,7 +4,7 @@ import com.saveourtool.save.frontend.externals.animations.* import csstype.Width import react.ChildrenBuilder -import react.dom.html.ReactHTML +import react.dom.html.ReactHTML.img import kotlinx.js.jso @@ -22,7 +22,7 @@ object BobPager : WelcomePager { } private fun ChildrenBuilder.renderAnimatedPage() { - ReactHTML.img { + img { style = jso { width = "140%".unsafeCast() } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfo.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfo.kt index b666f8f5ac..f09aaf3127 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfo.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfo.kt @@ -6,7 +6,8 @@ package com.saveourtool.save.frontend.components.views.welcome.pagers import csstype.* import react.ChildrenBuilder -import react.dom.html.ReactHTML +import react.dom.html.ReactHTML.div +import react.dom.html.ReactHTML.h1 import react.dom.html.ReactHTML.p import kotlinx.js.jso @@ -53,7 +54,7 @@ private const val CONTESTS_TEXT = """ * rendering of 4 paragraphs with info about SAVE */ fun ChildrenBuilder.renderGeneralInfoPage() { - ReactHTML.div { + div { style = jso { justifyContent = JustifyContent.center display = Display.flex @@ -63,7 +64,7 @@ fun ChildrenBuilder.renderGeneralInfoPage() { } className = ClassName("row") - ReactHTML.div { + div { style = jso { justifyContent = JustifyContent.center display = Display.flex @@ -75,7 +76,7 @@ fun ChildrenBuilder.renderGeneralInfoPage() { text("Easy CI", EASY_CI_TEXT) } - ReactHTML.div { + div { style = jso { justifyContent = JustifyContent.center display = Display.flex @@ -90,9 +91,9 @@ fun ChildrenBuilder.renderGeneralInfoPage() { } private fun ChildrenBuilder.text(title: String, textStr: String) { - ReactHTML.div { + div { className = ClassName("col-3 mx-3") - ReactHTML.h1 { + h1 { style = jso { textAlign = TextAlign.center } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfoPictures.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfoPictures.kt index 6b82d22361..c57e948159 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfoPictures.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/GeneralInfoPictures.kt @@ -9,8 +9,8 @@ import com.saveourtool.save.frontend.externals.animations.* import csstype.Color import csstype.Width import react.ChildrenBuilder -import react.dom.html.ReactHTML import react.dom.html.ReactHTML.h1 +import react.dom.html.ReactHTML.img import kotlinx.js.jso @@ -30,7 +30,7 @@ object GeneralInfoFirstPicture : WelcomePager { } +"Easy to start" } - ReactHTML.img { + img { style = jso { width = "65%".unsafeCast() } @@ -55,7 +55,7 @@ object GeneralInfoSecondPicture : WelcomePager { } +"User-friendly dashboards" } - ReactHTML.img { + img { style = jso { width = "130%".unsafeCast() } @@ -80,7 +80,7 @@ object GeneralInfoThirdPicture : WelcomePager { } +"Statistics for your tool" } - ReactHTML.img { + img { style = jso { width = "65%".unsafeCast() } @@ -105,7 +105,7 @@ object GeneralInfoFourthPicture : WelcomePager { } +"Build a team" } - ReactHTML.img { + img { style = jso { width = "135%".unsafeCast() } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/HighLevelSave.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/HighLevelSave.kt index 2d3b6c2dd0..228eeecdb5 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/HighLevelSave.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/components/views/welcome/pagers/HighLevelSave.kt @@ -2,7 +2,7 @@ package com.saveourtool.save.frontend.components.views.welcome.pagers import com.saveourtool.save.frontend.externals.animations.* import react.ChildrenBuilder -import react.dom.html.ReactHTML +import react.dom.html.ReactHTML.img @Suppress("CUSTOM_GETTERS_SETTERS") object HighLevelSave : WelcomePager { @@ -14,7 +14,7 @@ object HighLevelSave : WelcomePager { } private fun ChildrenBuilder.renderAnimatedPage() { - ReactHTML.img { + img { src = "img/save_hl.png" } } diff --git a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/externals/fontawesome/Icons.kt b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/externals/fontawesome/Icons.kt index 1bafe90874..4ad3251bce 100644 --- a/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/externals/fontawesome/Icons.kt +++ b/save-frontend/src/main/kotlin/com/saveourtool/save/frontend/externals/fontawesome/Icons.kt @@ -159,3 +159,11 @@ external val faChevronDown: FontAwesomeIconModule @JsModule("@fortawesome/free-solid-svg-icons/faCheckDouble") @JsNonModule external val faCheckDouble: FontAwesomeIconModule + +@JsModule("@fortawesome/free-solid-svg-icons/faTrophy") +@JsNonModule +external val faTrophy: FontAwesomeIconModule + +@JsModule("@fortawesome/free-solid-svg-icons/faCode") +@JsNonModule +external val faCode: FontAwesomeIconModule diff --git a/save-frontend/src/main/resources/img/green_square.png b/save-frontend/src/main/resources/img/green_square.png deleted file mode 100644 index 88ba74bf3f..0000000000 Binary files a/save-frontend/src/main/resources/img/green_square.png and /dev/null differ diff --git a/save-frontend/src/main/resources/img/undraw_certificate_re_yadi.svg b/save-frontend/src/main/resources/img/undraw_certificate_re_yadi.svg new file mode 100644 index 0000000000..3d30609a90 --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_certificate_re_yadi.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/undraw_exciting_news_re_y1iw.svg b/save-frontend/src/main/resources/img/undraw_exciting_news_re_y1iw.svg new file mode 100644 index 0000000000..c1eb65ed32 --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_exciting_news_re_y1iw.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/undraw_for_review_eqxk.svg b/save-frontend/src/main/resources/img/undraw_for_review_eqxk.svg new file mode 100644 index 0000000000..e24bd3e5cf --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_for_review_eqxk.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/undraw_mailbox_re_dvds.svg b/save-frontend/src/main/resources/img/undraw_mailbox_re_dvds.svg new file mode 100644 index 0000000000..892d73f18c --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_mailbox_re_dvds.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/undraw_news_re_6uub.svg b/save-frontend/src/main/resources/img/undraw_news_re_6uub.svg new file mode 100644 index 0000000000..7ea1aa0347 --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_news_re_6uub.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/undraw_notify_re_65on.svg b/save-frontend/src/main/resources/img/undraw_notify_re_65on.svg new file mode 100644 index 0000000000..83c85c8847 --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_notify_re_65on.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/undraw_selecting_team_re_ndkb.svg b/save-frontend/src/main/resources/img/undraw_selecting_team_re_ndkb.svg new file mode 100644 index 0000000000..ab662b3c4f --- /dev/null +++ b/save-frontend/src/main/resources/img/undraw_selecting_team_re_ndkb.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/save-frontend/src/main/resources/img/user.svg b/save-frontend/src/main/resources/img/user.svg deleted file mode 100644 index c650a40caf..0000000000 --- a/save-frontend/src/main/resources/img/user.svg +++ /dev/null @@ -1,270 +0,0 @@ - - - - diff --git a/save-frontend/src/main/resources/scss/_buttons.scss b/save-frontend/src/main/resources/scss/_buttons.scss index 599a4b2a23..f66189f50c 100644 --- a/save-frontend/src/main/resources/scss/_buttons.scss +++ b/save-frontend/src/main/resources/scss/_buttons.scss @@ -2,12 +2,6 @@ z-index: 0 } -.btn-primary { - color: #fff; - background-color: #53669c; - border-color: #53669c; -} - .btn-circle { border-radius: 100%; height: 2.5rem; diff --git a/save-orchestrator/build.gradle.kts b/save-orchestrator/build.gradle.kts index f7c8e14e01..41fb8fbfbf 100644 --- a/save-orchestrator/build.gradle.kts +++ b/save-orchestrator/build.gradle.kts @@ -85,6 +85,7 @@ dependencies { implementation(libs.zip4j) implementation(libs.spring.cloud.starter.kubernetes.client.config) implementation(libs.fabric8.kubernetes.client) + implementation(libs.spring.kafka) testImplementation(libs.fabric8.kubernetes.server.mock) } diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt index df3433bba4..4a86aa72b6 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/Utils.kt @@ -5,6 +5,7 @@ package com.saveourtool.save.orchestrator import com.saveourtool.save.orchestrator.config.ConfigProperties.AgentSettings +import com.saveourtool.save.orchestrator.runner.TEST_SUITES_DIR_NAME import com.github.dockerjava.api.DockerClient import com.github.dockerjava.api.async.ResultCallback @@ -113,6 +114,7 @@ internal fun fillAgentPropertiesFromConfiguration( "backend.url=${agentSettings.backendUrl}" line.startsWith("orchestratorUrl=") && agentSettings.orchestratorUrl != null -> "orchestratorUrl=${agentSettings.orchestratorUrl}" + line.startsWith("testSuitesDir=") -> "testSuitesDir=$TEST_SUITES_DIR_NAME" else -> line } } diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/config/KafkaConfiguration.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/config/KafkaConfiguration.kt new file mode 100644 index 0000000000..9a858dd6e9 --- /dev/null +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/config/KafkaConfiguration.kt @@ -0,0 +1,164 @@ +package com.saveourtool.save.orchestrator.config + +import com.saveourtool.save.kafka.TestExecutionTaskDto +import com.saveourtool.save.orchestrator.kafka.AgentKafkaListener +import com.saveourtool.save.orchestrator.kafka.KafkaSender +import org.apache.kafka.clients.admin.NewTopic +import org.apache.kafka.clients.consumer.ConsumerConfig +import org.apache.kafka.clients.consumer.CooperativeStickyAssignor +import org.apache.kafka.clients.producer.ProducerConfig +import org.apache.kafka.clients.producer.internals.DefaultPartitioner +import org.apache.kafka.common.serialization.StringDeserializer +import org.apache.kafka.common.serialization.StringSerializer +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Value +import org.springframework.boot.autoconfigure.kafka.KafkaProperties +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.Profile +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory +import org.springframework.kafka.core.* +import org.springframework.kafka.listener.DeadLetterPublishingRecoverer +import org.springframework.kafka.listener.SeekToCurrentErrorHandler +import org.springframework.kafka.support.serializer.ErrorHandlingDeserializer +import org.springframework.kafka.support.serializer.JsonDeserializer +import org.springframework.kafka.support.serializer.JsonSerializer +import java.util.* + +/** + * Kafka producer and consumer configuration + * + * @property kafkaProperties + */ +@Configuration +@Profile("dev & kafka") +internal class KafkaConfiguration( + private val kafkaProperties: KafkaProperties) { + /** + * @property requestTopicName + */ + @Value("\${kafka.test.execution.request.topic.name}") + lateinit var requestTopicName: String + + /** + * @property requestTopicNameDlt + */ + @Value("\${kafka.test.execution.request.topic.name}.DLT") + lateinit var requestTopicNameDlt: String + + /** + * @property responseTopicName + */ + @Value("\${kafka.test.execution.response.topic.name}") + lateinit var responseTopicName: String + + /** + * @property consumerGroup + */ + @Value("\${spring.kafka.consumer.group-id}") + lateinit var consumerGroup: String + + /** + * @return kafka producer properties + */ + @Bean + fun producerConfig(): Map { + val producerProps = HashMap(kafkaProperties.buildProducerProperties()) + producerProps[ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG] = StringSerializer::class.java as Object + producerProps[ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG] = JsonSerializer::class.java as Object + producerProps[ProducerConfig.PARTITIONER_CLASS_CONFIG] = DefaultPartitioner::class.java as Object + return producerProps + } + + /** + * @return kafka producer factory + */ + @Bean + fun producerFactory(): ProducerFactory = DefaultKafkaProducerFactory(producerConfig()) + + /** + * @return kafka template + */ + @Bean + fun kafkaTemplate(): KafkaTemplate = KafkaTemplate(producerFactory()) + + /** + * @return kafka consumer properties + */ + @Bean + fun consumerConfig(): Map { + val consumerProps = HashMap(kafkaProperties.buildConsumerProperties()) + consumerProps[ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG] = "true" + consumerProps[ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG] = "10" + consumerProps[ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG] = "60000" + consumerProps[ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG] = ErrorHandlingDeserializer::class.java + consumerProps[ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG] = ErrorHandlingDeserializer::class.java + consumerProps[ErrorHandlingDeserializer.KEY_DESERIALIZER_CLASS] = StringDeserializer::class.java + consumerProps[ErrorHandlingDeserializer.VALUE_DESERIALIZER_CLASS] = JsonDeserializer::class.java + consumerProps[ConsumerConfig.PARTITION_ASSIGNMENT_STRATEGY_CONFIG] = listOf(CooperativeStickyAssignor::class.java) + consumerProps[JsonDeserializer.TRUSTED_PACKAGES] = "*" + return consumerProps + } + + /** + * @return consumer factory + */ + @Bean + fun consumerFactory(): ConsumerFactory = DefaultKafkaConsumerFactory(consumerConfig()) + + /** + * @param template + * @return kafka listener container factory + */ + @Suppress("GENERIC_VARIABLE_WRONG_DECLARATION") + @Bean + fun kafkaListenerContainerFactory(template: KafkaTemplate): ConcurrentKafkaListenerContainerFactory { + val factory = ConcurrentKafkaListenerContainerFactory() + factory.setConsumerFactory(consumerFactory()) + factory.setConcurrency(1) + factory.getContainerProperties().setPollTimeout(LISTENER_POLL_TIMEOUT) + factory.setErrorHandler( + SeekToCurrentErrorHandler( + DeadLetterPublishingRecoverer(template) + ) + ) + return factory + } + + /** + * @return kafka agent tasks topic bean + */ + @Bean + fun saveAgentTasks(): NewTopic { + log.info("Create topic: $requestTopicName.") + return NewTopic(requestTopicName, Optional.empty(), Optional.empty()) + } + + /** + * @param template + * @return test execution sender + */ + @Bean + fun testExecutionSender(template: KafkaTemplate): KafkaSender { + log.info("Create sender for {} topic.", requestTopicName) + return KafkaSender(template as KafkaTemplate, requestTopicName) + } + + /** + * @return agentKafkaListener + */ + @Bean + fun agentKafkaListener(): AgentKafkaListener { + log.info("Create listener for {} topic.", responseTopicName) + return AgentKafkaListener(requestTopicName, consumerGroup) + } + + companion object { + private val log = LoggerFactory.getLogger(KafkaConfiguration::class.java) + + /** + * @property listenerPollTimeout + */ + private const val LISTENER_POLL_TIMEOUT = 500L + } +} diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/AgentsController.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/AgentsController.kt index 231a865282..26050de123 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/AgentsController.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/AgentsController.kt @@ -1,25 +1,21 @@ package com.saveourtool.save.orchestrator.controller -import com.saveourtool.save.domain.FileKey import com.saveourtool.save.entities.Agent import com.saveourtool.save.entities.Execution import com.saveourtool.save.execution.ExecutionStatus import com.saveourtool.save.orchestrator.BodilessResponseEntity import com.saveourtool.save.orchestrator.config.ConfigProperties -import com.saveourtool.save.orchestrator.runner.TEST_SUITES_DIR_NAME import com.saveourtool.save.orchestrator.service.AgentService import com.saveourtool.save.orchestrator.service.DockerService import com.saveourtool.save.orchestrator.utils.LoggingContextImpl -import com.saveourtool.save.orchestrator.utils.allExecute -import com.saveourtool.save.orchestrator.utils.tryMarkAsExecutable -import com.saveourtool.save.utils.* +import com.saveourtool.save.utils.debug +import com.saveourtool.save.utils.info import com.github.dockerjava.api.exception.DockerClientException import com.github.dockerjava.api.exception.DockerException import io.fabric8.kubernetes.client.KubernetesClientException import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Qualifier -import org.springframework.core.io.buffer.DataBuffer import org.springframework.http.HttpStatus import org.springframework.http.MediaType import org.springframework.http.ResponseEntity @@ -32,19 +28,14 @@ import org.springframework.web.bind.annotation.RequestPart import org.springframework.web.bind.annotation.RestController import org.springframework.web.reactive.function.client.WebClient import org.springframework.web.reactive.function.client.WebClientResponseException -import org.springframework.web.reactive.function.client.bodyToFlux import org.springframework.web.server.ResponseStatusException import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.doOnError -import reactor.kotlin.core.publisher.toFlux import java.io.File import java.io.FileOutputStream -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.Paths -import kotlin.io.path.* +import kotlin.io.path.absolutePathString /** * Controller used to start agents with needed information @@ -56,15 +47,6 @@ class AgentsController( private val configProperties: ConfigProperties, @Qualifier("webClientBackend") private val webClientBackend: WebClient, ) { - // Somehow simple path.createDirectories() doesn't work on macOS, probably due to Apple File System features - private val tmpDir = Paths.get(configProperties.testResources.tmpPath).let { - if (it.exists()) { - it - } else { - it.createDirectories() - } - } - /** * Schedules tasks to build base images, create a number of containers and put their data into the database. * @@ -89,22 +71,10 @@ class AgentsController( "status=${execution.status}]" } Mono.fromCallable { - createTempDirectory( - directory = tmpDir, - prefix = "save-execution-${execution.id}" - ) + // todo: pass SDK via request body + dockerService.prepareConfiguration(execution) } - .flatMap { resourcesForExecution -> - val resourcesPath = resourcesForExecution.resolve(TEST_SUITES_DIR_NAME) - execution.downloadTestsTo(resourcesPath) - .then(execution.downloadAdditionalFilesTo(resourcesPath)) - .thenReturn(resourcesForExecution) - } - .publishOn(agentService.scheduler) - .map { - // todo: pass SDK via request body - dockerService.prepareConfiguration(it, execution) - } + .subscribeOn(agentService.scheduler) .onErrorResume({ it is DockerException || it is DockerClientException }) { dex -> reportExecutionError(execution, "Unable to build image and containers", dex) } @@ -137,110 +107,6 @@ class AgentsController( } } - private fun unzipIfRequired( - pathToFile: Path, - ) { - // FixMe: for now support only .zip files - if (pathToFile.name.endsWith(".zip")) { - val shouldBeExecutable = Files.getPosixFilePermissions(pathToFile).any { allExecute.contains(it) } - pathToFile.extractZipHereSafely() - if (shouldBeExecutable) { - log.info { "Marking files in ${pathToFile.parent} executable..." } - Files.walk(pathToFile.parent) - .filter { it.isRegularFile() } - .forEach { with(loggingContext) { it.tryMarkAsExecutable() } } - } - Files.delete(pathToFile) - } - } - - @Suppress("TooGenericExceptionCaught") - private fun Path.extractZipHereSafely() { - try { - extractZipHere() - } catch (e: Exception) { - log.error("Error occurred during extracting of archive $name", e) - } - } - - private fun Execution.downloadAdditionalFilesTo( - targetDirectory: Path - ): Mono = parseAndGetAdditionalFiles() - .toFlux() - .flatMap { fileKey -> - val pathToFile = targetDirectory.resolve(fileKey.name) - (fileKey to this).downloadTo(pathToFile) - .map { unzipIfRequired(pathToFile) } - } - .collectList() - .map { - log.info { "Downloaded all additional files for execution $id to $targetDirectory" } - } - .lazyDefaultIfEmpty { - log.warn { - "Not found any additional files for execution $id" - } - } - - private fun Pair.downloadTo( - pathToFile: Path - ): Mono = this.let { (fileKey, execution) -> - webClientBackend.post() - .uri("/files/{organizationName}/{projectName}/download", execution.project.organization.name, execution.project.name) - .contentType(MediaType.APPLICATION_JSON) - .bodyValue(fileKey) - .accept(MediaType.APPLICATION_OCTET_STREAM) - .retrieve() - .bodyToFlux() - .let { content -> - pathToFile.parent.createDirectories() - content.writeTo(pathToFile) - } - .then( - Mono.fromCallable { - // TODO: need to store information about isExecutable in Execution (FileKey) - with(loggingContext) { pathToFile.tryMarkAsExecutable() } - log.debug { - "Downloaded $fileKey to ${pathToFile.absolutePathString()}" - } - } - ) - .lazyDefaultIfEmpty { - log.warn { - "Not found additional file $fileKey for execution ${execution.id}" - } - } - } - - private fun Execution.downloadTestsTo( - targetDirectory: Path - ): Mono = webClientBackend.post() - .uri( - "/test-suites-sources/download-snapshot-by-execution-id?executionId={executionId}", - requiredId(), - ) - .contentType(MediaType.APPLICATION_JSON) - .accept(MediaType.APPLICATION_OCTET_STREAM) - .retrieve() - .bodyToFlux() - .let { content -> - targetDirectory.createDirectories() - val targetFile = Files.createTempFile(targetDirectory, "archive-", ARCHIVE_EXTENSION) - content.writeTo(targetFile) - } - .map { - it.extractZipHere() - it.deleteExisting() - } - .map { - log.info { "Downloaded all tests for execution $id to $targetDirectory" } - } - .lazyDefaultIfEmpty { - log.warn { - "Not found any tests for execution ${requiredId()}" - } - } - private fun reportExecutionError( execution: Execution, failReason: String, diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/KafkaController.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/KafkaController.kt new file mode 100644 index 0000000000..6bf735e55b --- /dev/null +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/controller/KafkaController.kt @@ -0,0 +1,29 @@ +package com.saveourtool.save.orchestrator.controller + +import com.saveourtool.save.kafka.TestExecutionTaskDto +import com.saveourtool.save.orchestrator.kafka.KafkaSender +import com.saveourtool.save.v1 +import org.springframework.context.annotation.Profile +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +/** + * Controller for working with contests. + */ +@Profile("dev & kafka") +@RestController +@RequestMapping(path = ["/api/$v1/kafka"]) +internal class KafkaController( + private val testExecutionSender: KafkaSender +) { + /** + * @param task + * @return Organization + */ + @PostMapping("/sendTestExecutionTask") + fun sendTestExecutionTask(@RequestBody task: TestExecutionTaskDto) { + testExecutionSender.sendMessage(task) + } +} diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/docker/DockerAgentRunner.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/docker/DockerAgentRunner.kt index 7d3a4ae239..6aff8a0f9b 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/docker/DockerAgentRunner.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/docker/DockerAgentRunner.kt @@ -31,6 +31,8 @@ import java.util.concurrent.ConcurrentMap import kotlin.io.path.createTempDirectory import kotlin.io.path.writeText +import kotlin.math.abs +import kotlin.random.Random /** * [AgentRunner] that uses Docker Daemon API to run save-agents @@ -53,10 +55,8 @@ class DockerAgentRunner( executionId: Long, configuration: DockerService.RunConfiguration, replicas: Int, - workingDir: String, ): List { - val (baseImageTag, agentRunCmd, pvId) = configuration - require(pvId is DockerPvId) { "${DockerPersistentVolumeService::class.simpleName} can only operate with ${DockerPvId::class.simpleName}" } + require(configuration.pvId is DockerPvId) { "${DockerPersistentVolumeService::class.simpleName} can only operate with ${DockerPvId::class.simpleName}" } logger.debug { "Pulling image ${configuration.imageTag}" } dockerClient.pullImageCmd(configuration.imageTag) @@ -66,7 +66,7 @@ class DockerAgentRunner( return (1..replicas).map { number -> logger.info("Creating a container #$number for execution.id=$executionId") - createContainerFromImage(baseImageTag, pvId, workingDir, agentRunCmd, containerName("$executionId-$number")).also { agentId -> + createContainerFromImage(configuration as DockerService.RunConfiguration, containerName("$executionId-$number")).also { agentId -> logger.info("Created a container id=$agentId for execution.id=$executionId") agentIdsByExecution .getOrPut(executionId) { mutableListOf() } @@ -166,23 +166,23 @@ class DockerAgentRunner( * @throws RuntimeException if an exception not specific to docker has occurred */ @Suppress("UnsafeCallOnNullableType", "TOO_LONG_FUNCTION") - private fun createContainerFromImage(baseImageTag: String, - pvId: DockerPvId, - workingDir: String, - runCmd: List, + private fun createContainerFromImage(configuration: DockerService.RunConfiguration, containerName: String, ): String { + val baseImageTag = configuration.imageTag + val pvId = configuration.pvId + val runCmd = configuration.runCmd val envFileTargetPath = "$SAVE_AGENT_USER_HOME/.env" // createContainerCmd accepts image name, not id, so we retrieve it from tags val createContainerCmdResponse: CreateContainerResponse = dockerClient.createContainerCmd(baseImageTag) - .withWorkingDir(workingDir) + .withWorkingDir(EXECUTION_DIR) // Load environment variables required by save-agent and then run it. // Rely on `runCmd` format: last argument is parameter of the subshell. .withCmd( // this part is like `sh -c` with probably some other flags runCmd.dropLast(1) + ( // last element is an actual command that will be executed in a new shell - "env \$(cat $envFileTargetPath | xargs) sh -c \"${runCmd.last()}\"" + """env $(cat $envFileTargetPath | xargs) sh -c "cp $SAVE_AGENT_USER_HOME/resources/* . && ${runCmd.last()}"""" ) ) .withName(containerName) @@ -191,7 +191,7 @@ class DockerAgentRunner( HostConfig.newHostConfig() .withBinds(Bind( // Apparently, target path needs to be wrapped into [Volume] object in Docker API. - pvId.volumeName, Volume(EXECUTION_DIR) + pvId.volumeName, Volume("$SAVE_AGENT_USER_HOME/resources") )) .withRuntime(settings.runtime) // processes from inside the container will be able to access host's network using this hostname @@ -216,7 +216,10 @@ class DockerAgentRunner( val containerId = createContainerCmdResponse.id val envFile = createTempDirectory("orchestrator").resolve(".env").apply { writeText(""" - AGENT_ID=$containerId""".trimIndent() + AGENT_ID=$containerId + EXECUTION_ID=${configuration.resourcesConfiguration.executionId} + ADDITIONAL_FILES_LIST=${configuration.resourcesConfiguration.additionalFilesString} + """.trimIndent() ) } copyResourcesIntoContainer( @@ -254,4 +257,5 @@ class DockerAgentRunner( /** * @param id */ -private fun containerName(id: String) = "save-execution-$id" +@Suppress("MAGIC_NUMBER", "MagicNumber") +private fun containerName(id: String) = "save-execution-$id-${abs(Random.nextInt(100, 999))}" diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kafka/AgentKafkaListener.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kafka/AgentKafkaListener.kt new file mode 100644 index 0000000000..831ac0c52a --- /dev/null +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kafka/AgentKafkaListener.kt @@ -0,0 +1,47 @@ +package com.saveourtool.save.orchestrator.kafka + +import com.saveourtool.save.kafka.TestExecutionTaskDto +import org.slf4j.LoggerFactory +import org.springframework.kafka.annotation.KafkaHandler +import org.springframework.kafka.annotation.KafkaListener +import org.springframework.kafka.support.KafkaHeaders +import org.springframework.messaging.handler.annotation.Header +import org.springframework.messaging.handler.annotation.Payload + +/** + * @property topic + * @property groupId + */ +@KafkaListener(id = "#{__listener.topic}.listener", topics = ["#{__listener.topic}"], groupId = "#{__listener.groupId}") +class AgentKafkaListener( + val topic: String, + val groupId: String +) { + /** + * @param data + * @param messageId + * @param partition + * @param topic + * @param ts + */ + @KafkaHandler + fun listen( + @Payload data: TestExecutionTaskDto?, + @Header(KafkaHeaders.RECEIVED_MESSAGE_KEY) messageId: String?, + @Header(KafkaHeaders.RECEIVED_PARTITION_ID) partition: Int, + @Header(KafkaHeaders.RECEIVED_TOPIC) topic: String?, + @Header(KafkaHeaders.RECEIVED_TIMESTAMP) ts: Long + ) { + log.info( + "Received request. messageId: {} , partition: {} , topic: {}, ts: {}, payload: {}", + messageId, + partition, + topic, + ts, + data + ) + } + companion object { + private val log = LoggerFactory.getLogger(AgentKafkaListener::class.java) + } +} diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kafka/KafkaSender.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kafka/KafkaSender.kt new file mode 100644 index 0000000000..6cd955e8df --- /dev/null +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kafka/KafkaSender.kt @@ -0,0 +1,94 @@ +package com.saveourtool.save.orchestrator.kafka + +import com.saveourtool.save.kafka.KafkaMsg +import org.apache.kafka.clients.producer.ProducerRecord +import org.apache.kafka.common.header.internals.RecordHeader +import org.slf4j.LoggerFactory +import org.springframework.kafka.core.KafkaTemplate +import org.springframework.kafka.requestreply.CorrelationKey +import org.springframework.kafka.support.KafkaHeaders +import org.springframework.kafka.support.SendResult +import org.springframework.util.concurrent.ListenableFuture +import org.springframework.util.concurrent.ListenableFutureCallback +import java.util.concurrent.atomic.AtomicLong + +/** + * Kafka sender + */ +class KafkaSender( + private val template: KafkaTemplate, + private val topic: String +) { + /** + * @param msg + * @return send result future + */ + @Suppress("TYPE_ALIAS", "GENERIC_VARIABLE_WRONG_DECLARATION") + fun sendMessage(msg: V): ListenableFuture> { + val messageId = msg.messageId + log.info("Sending a message with id {} to topic: {}", messageId, topic) + log.debug("The message to be sent is: {}", msg) + val kafkaRecord = ProducerRecord(topic, messageId, msg) + return template.send(kafkaRecord) + } + + /** + * @param topic + * @param partition + * @param corrKey + * @param msg + * @return send result future + */ + @Suppress("TYPE_ALIAS") + fun sendMessage( + topic: String?, + partition: Int, + corrKey: CorrelationKey?, + msg: V + ): ListenableFuture> { + val messageId = msg.messageId + log.info( + "Sending a message with id $messageId and correlation key to topic: $topic partition $partition" + ) + log.debug("The message to be sent is: $msg") + val kafkaRecord: ProducerRecord = ProducerRecord(topic, partition, messageId, msg) + corrKey?.let { + kafkaRecord.headers().add(RecordHeader(KafkaHeaders.CORRELATION_ID, corrKey.getCorrelationId())) + } + return template.send(kafkaRecord) + } + + /** + * @param msg + * @param successCount + * @param failedRecords + */ + @Suppress("TYPE_ALIAS", "TooGenericExceptionCaught") + fun sendMessage(msg: V, successCount: AtomicLong, failedRecords: MutableSet) { + val messageId = msg.messageId + try { + val future: ListenableFuture> = sendMessage(msg) + future.addCallback(object : ListenableFutureCallback> { + override fun onSuccess(result: SendResult?) { + log.info( + "Msg id={} was successfully sent with offset: {}", + result?.getProducerRecord()?.key(), result?.getRecordMetadata()?.offset() + ) + successCount.incrementAndGet() + } + + override fun onFailure(ex: Throwable) { + log.error("Unable to send message=[{}].", messageId, ex) + failedRecords.add(messageId) + } + }) + } catch (e: Exception) { + log.error("Unable to send task: {}", messageId, e) + failedRecords.add(messageId) + } + } + + companion object { + private val log = LoggerFactory.getLogger(KafkaSender::class.java) + } +} diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt index 1401bfb742..9cab6a4d8d 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/kubernetes/KubernetesManager.kt @@ -3,7 +3,6 @@ package com.saveourtool.save.orchestrator.kubernetes import com.saveourtool.save.orchestrator.config.ConfigProperties import com.saveourtool.save.orchestrator.runner.AgentRunner import com.saveourtool.save.orchestrator.runner.AgentRunnerException -import com.saveourtool.save.orchestrator.runner.EXECUTION_DIR import com.saveourtool.save.orchestrator.runner.SAVE_AGENT_USER_HOME import com.saveourtool.save.orchestrator.service.DockerService import com.saveourtool.save.orchestrator.service.PersistentVolumeId @@ -40,9 +39,11 @@ class KubernetesManager( override fun create(executionId: Long, configuration: DockerService.RunConfiguration, replicas: Int, - workingDir: String, ): List { - val (baseImageTag, agentRunCmd, pvId) = configuration + val baseImageTag = configuration.imageTag + val agentRunCmd = configuration.runCmd + val pvId = configuration.pvId + val workingDir = configuration.workingDir require(pvId is KubernetesPvId) { "${KubernetesPersistentVolumeService::class.simpleName} can only operate with ${KubernetesPvId::class.simpleName}" } requireNotNull(configProperties.kubernetes) // fixme: pass image name instead of ID from the outside @@ -79,7 +80,7 @@ class KubernetesManager( restartPolicy = "Never" initContainers = initContainersSpec(pvId) containers = listOf( - agentContainerSpec(baseImageTag, agentRunCmd) + agentContainerSpec(baseImageTag, agentRunCmd, workingDir, configuration.resourcesConfiguration) ) volumes = listOf( Volume().apply { @@ -203,12 +204,13 @@ class KubernetesManager( Container().apply { name = "save-vol-copier" image = "alpine:latest" + val targetDir = configProperties.kubernetes.pvcMountPath command = listOf( "sh", "-c", - "if [ -z \"$(ls -A $EXECUTION_DIR)\" ];" + - " then mkdir -p $EXECUTION_DIR && cp -R ${pvId.sourcePath}/* $EXECUTION_DIR" + - " && chown -R 1100:1100 $EXECUTION_DIR && echo Successfully copied;" + - " else echo Copying already in progress && ls -A $EXECUTION_DIR && sleep $waitForCopySeconds;" + + "if [ -z \"$(ls -A $targetDir)\" ];" + + " then mkdir -p $targetDir && cp -R ${pvId.sourcePath}/* $targetDir" + + " && chown -R 1100:1100 $targetDir && echo Successfully copied;" + + " else echo Copying already in progress && ls -A $targetDir && sleep $waitForCopySeconds;" + " fi" ) volumeMounts = listOf( @@ -225,7 +227,13 @@ class KubernetesManager( ) } - private fun agentContainerSpec(imageName: String, agentRunCmd: List) = Container().apply { + @Suppress("TOO_LONG_FUNCTION") + private fun agentContainerSpec( + imageName: String, + agentRunCmd: List, + workingDir: String, + resourcesConfiguration: DockerService.RunConfiguration.ResourcesConfiguration, + ) = Container().apply { name = "save-agent-pod" image = imageName imagePullPolicy = "IfNotPresent" // so that local images could be used @@ -237,17 +245,26 @@ class KubernetesManager( fieldPath = "metadata.name" } } - } + }, + EnvVar().apply { + name = "EXECUTION_ID" + value = "${resourcesConfiguration.executionId}" + }, + EnvVar().apply { + name = "ADDITIONAL_FILES_LIST" + value = resourcesConfiguration.additionalFilesString + }, ) + val resourcesPath = requireNotNull(configProperties.kubernetes).pvcMountPath this.command = agentRunCmd.dropLast(1) - this.args = listOf(agentRunCmd.last()) + this.args = listOf("cp $resourcesPath/* . && ${agentRunCmd.last()}") this.workingDir = workingDir volumeMounts = listOf( VolumeMount().apply { name = "save-execution-pvc" - mountPath = requireNotNull(configProperties.kubernetes).pvcMountPath + mountPath = resourcesPath } ) } diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/runner/AgentRunner.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/runner/AgentRunner.kt index daf39bdd08..efe2ee99d9 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/runner/AgentRunner.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/runner/AgentRunner.kt @@ -17,14 +17,12 @@ interface AgentRunner { * @param executionId and ID of execution for which agents will run tests * @param configuration [DockerService.RunConfiguration] for the created containers * @param replicas number of agents acting in parallel - * @param workingDir execution directory inside the container * @return unique identifier of created instances that can be used to manipulate them later */ fun create( executionId: Long, configuration: DockerService.RunConfiguration, replicas: Int, - workingDir: String, ): List /** diff --git a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/service/DockerService.kt b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/service/DockerService.kt index 4a0cee82fc..2db764a17b 100644 --- a/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/service/DockerService.kt +++ b/save-orchestrator/src/main/kotlin/com/saveourtool/save/orchestrator/service/DockerService.kt @@ -31,6 +31,7 @@ import org.springframework.web.reactive.function.client.bodyToMono import reactor.core.publisher.Flux import java.nio.file.Path +import java.nio.file.Paths import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicLong @@ -51,6 +52,15 @@ class DockerService( private val persistentVolumeService: PersistentVolumeService, private val agentService: AgentService, ) { + // Somehow simple path.createDirectories() doesn't work on macOS, probably due to Apple File System features + private val tmpDir = Paths.get(configProperties.testResources.tmpPath).let { + if (it.exists()) { + it + } else { + it.createDirectories() + } + } + @Suppress("NonBooleanPropertyPrefixedWithIs") private val isAgentStoppingInProgress = AtomicBoolean(false) @@ -61,16 +71,18 @@ class DockerService( /** * Function that builds a base image with test resources * - * @param resourcesForExecution location to resources are required for [execution] * @param execution [Execution] from which this workflow is started * @return image ID and execution command for the agent * @throws DockerException if interaction with docker daemon is not successful */ @Suppress("UnsafeCallOnNullableType") - fun prepareConfiguration(resourcesForExecution: Path, execution: Execution): RunConfiguration { + fun prepareConfiguration(execution: Execution): RunConfiguration { + val resourcesForExecution = createTempDirectory( + directory = tmpDir, + prefix = "save-execution-${execution.id}" + ) log.info("Preparing volume for execution.id=${execution.id}") val buildResult = prepareImageAndVolumeForExecution(resourcesForExecution, execution) - // todo (k8s): need to also push it so that other nodes will have access to it log.info("For execution.id=${execution.id} using base image [${buildResult.imageTag}] and PV [id=${buildResult.pvId}]") return buildResult } @@ -89,7 +101,6 @@ class DockerService( executionId = executionId, configuration = configuration, replicas = configProperties.agentsCount, - workingDir = EXECUTION_DIR, ) /** @@ -239,6 +250,10 @@ class DockerService( runCmd = listOf("sh", "-c", "chmod +x $SAVE_AGENT_EXECUTABLE_NAME && ./$SAVE_AGENT_EXECUTABLE_NAME"), pvId = pvId, resourcesPath = resourcesForExecution, + resourcesConfiguration = RunConfiguration.ResourcesConfiguration( + executionId = execution.requiredId(), + additionalFilesString = execution.additionalFiles, + ), ) } @@ -263,13 +278,26 @@ class DockerService( * Usually looks like `sh -c "rest of the command"`. * @property pvId ID of a persistent volume that should be attached to a container * @property resourcesPath FixMe: needed only until agents download test and additional files by themselves + * @property workingDir + * @property resourcesConfiguration */ data class RunConfiguration( val imageTag: String, val runCmd: List, val pvId: I, + val workingDir: String = EXECUTION_DIR, val resourcesPath: Path, - ) + val resourcesConfiguration: ResourcesConfiguration, + ) { + /** + * @property executionId + * @property additionalFilesString + */ + data class ResourcesConfiguration( + val executionId: Long, + val additionalFilesString: String, + ) + } companion object { private val log = LoggerFactory.getLogger(DockerService::class.java) diff --git a/save-orchestrator/src/main/resources/application-kafka.properties b/save-orchestrator/src/main/resources/application-kafka.properties new file mode 100644 index 0000000000..364dbb76a5 --- /dev/null +++ b/save-orchestrator/src/main/resources/application-kafka.properties @@ -0,0 +1,5 @@ +spring.kafka.bootstrap-servers=localhost:29092 +spring.kafka.consumer.group-id=orchestrator-local-group +spring.kafka.consumer.auto-offset-reset=earliest +kafka.test.execution.request.topic.name=save.test.execution.tasks +kafka.test.execution.response.topic.name=save.test.execution.tasks \ No newline at end of file diff --git a/save-orchestrator/src/main/resources/application.properties b/save-orchestrator/src/main/resources/application.properties index f21f326c06..6333e90ec5 100644 --- a/save-orchestrator/src/main/resources/application.properties +++ b/save-orchestrator/src/main/resources/application.properties @@ -16,5 +16,5 @@ orchestrator.agents-count=3 orchestrator.agents-heart-beat-timeout-millis=60000 orchestrator.heart-beat-inspector-interval=10 orchestrator.agent-settings.agent-id-env=AGENT_ID -orchestrator.agentsStartTimeoutMillis=60000 -orchestrator.agentsStartCheckIntervalMillis=10000 +orchestrator.agents-start-timeout-millis=60000 +orchestrator.agents-start-check-interval-millis=10000 diff --git a/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/controller/agents/AgentsControllerTest.kt b/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/controller/agents/AgentsControllerTest.kt index ab1f9c3d08..d9dad477e4 100644 --- a/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/controller/agents/AgentsControllerTest.kt +++ b/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/controller/agents/AgentsControllerTest.kt @@ -9,6 +9,7 @@ import com.saveourtool.save.orchestrator.config.ConfigProperties import com.saveourtool.save.orchestrator.controller.AgentsController import com.saveourtool.save.orchestrator.docker.DockerPvId import com.saveourtool.save.orchestrator.runner.AgentRunner +import com.saveourtool.save.orchestrator.runner.EXECUTION_DIR import com.saveourtool.save.orchestrator.service.AgentService import com.saveourtool.save.orchestrator.service.DockerService import com.saveourtool.save.testutils.checkQueues @@ -71,7 +72,7 @@ class AgentsControllerTest { } @Test - @Suppress("TOO_LONG_FUNCTION") + @Suppress("TOO_LONG_FUNCTION", "LongMethod", "UnsafeCallOnNullableType") fun `should build image, query backend and start containers`() { val project = Project.stub(null) val execution = Execution.stub(project).apply { @@ -92,12 +93,17 @@ class AgentsControllerTest { .addHeader("Content-Type", "application/octet-stream") .setBody(Buffer().readFrom(tmpArchive.inputStream())) ) - whenever(dockerService.prepareConfiguration(any(), any())).thenReturn( + whenever(dockerService.prepareConfiguration(any())).thenReturn( DockerService.RunConfiguration( - "test-image-id", - listOf("sh", "-c", "test-exec-cmd"), - DockerPvId("test-pv-id"), - Path.of("test-resources-path"), + imageTag = "test-image-id", + runCmd = listOf("sh", "-c", "test-exec-cmd"), + pvId = DockerPvId("test-pv-id"), + workingDir = EXECUTION_DIR, + resourcesPath = Path.of("test-resources-path"), + resourcesConfiguration = DockerService.RunConfiguration.ResourcesConfiguration( + executionId = execution.id!!, + additionalFilesString = "", + ) ) ) whenever(dockerService.createContainers(any(), any())) @@ -122,7 +128,7 @@ class AgentsControllerTest { .expectStatus() .isAccepted Thread.sleep(2_500) // wait for background task to complete on mocks - verify(dockerService).prepareConfiguration(any(), any()) + verify(dockerService).prepareConfiguration(any()) verify(dockerService).createContainers(any(), any()) verify(dockerService).startContainersAndUpdateExecution(any(), anyList()) diff --git a/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/docker/DockerContainerManagerTest.kt b/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/docker/DockerContainerManagerTest.kt index ce3ef600ac..32ebb88278 100644 --- a/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/docker/DockerContainerManagerTest.kt +++ b/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/docker/DockerContainerManagerTest.kt @@ -61,10 +61,14 @@ class DockerContainerManagerTest { baseImage.repoTags.first(), listOf("bash", "-c", "./script.sh"), DockerPvId("test-volume"), - Path.of("test-resources-path"), + workingDir = "/", + resourcesPath = Path.of("test-resources-path"), + resourcesConfiguration = DockerService.RunConfiguration.ResourcesConfiguration( + executionId = 99L, + additionalFilesString = "", + ) ), replicas = 1, - workingDir = "/", ).single() val inspectContainerResponse = dockerClient .inspectContainerCmd(testContainerId) @@ -72,11 +76,11 @@ class DockerContainerManagerTest { Assertions.assertEquals("bash", inspectContainerResponse.path) Assertions.assertArrayEquals( - arrayOf("-c", "env \$(cat /home/save-agent/.env | xargs) sh -c \"./script.sh\""), + arrayOf("-c", "env \$(cat /home/save-agent/.env | xargs) sh -c \"cp /home/save-agent/resources/* . && ./script.sh\""), inspectContainerResponse.args ) // leading extra slash: https://github.com/moby/moby/issues/6705 - Assertions.assertEquals("/save-execution-42-1", inspectContainerResponse.name) + Assertions.assertTrue(inspectContainerResponse.name.startsWith("/save-execution-42")) val resourceFile = createTempFile().toFile() resourceFile.writeText("Lorem ipsum dolor sit amet") diff --git a/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/service/DockerServiceTest.kt b/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/service/DockerServiceTest.kt index a7c851492d..fd47dfea58 100644 --- a/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/service/DockerServiceTest.kt +++ b/save-orchestrator/src/test/kotlin/com/saveourtool/save/orchestrator/service/DockerServiceTest.kt @@ -6,7 +6,6 @@ import com.saveourtool.save.orchestrator.config.Beans import com.saveourtool.save.orchestrator.config.ConfigProperties import com.saveourtool.save.orchestrator.docker.DockerAgentRunner import com.saveourtool.save.orchestrator.docker.DockerPersistentVolumeService -import com.saveourtool.save.orchestrator.runner.TEST_SUITES_DIR_NAME import com.saveourtool.save.orchestrator.testutils.TestConfiguration import com.saveourtool.save.testutils.checkQueues import com.saveourtool.save.testutils.cleanup @@ -58,7 +57,6 @@ class DockerServiceTest { @Autowired private lateinit var dockerClient: DockerClient @Autowired private lateinit var dockerService: DockerService @Autowired private lateinit var configProperties: ConfigProperties - private lateinit var testImageId: String private lateinit var testContainerId: String @BeforeEach @@ -86,13 +84,7 @@ class DockerServiceTest { .setHeader("Content-Type", "application/json") .setBody(Json.encodeToString(listOf("Test1", "Test2"))) ) - val tmpDir = Paths.get(configProperties.testResources.tmpPath).createDirectories() - val resourcesForExecution = createTempDirectory( - directory = tmpDir, - prefix = "save-execution-${testExecution.requiredId()}" - ) - resourcesForExecution.resolve(TEST_SUITES_DIR_NAME).createDirectory() - val configuration = dockerService.prepareConfiguration(resourcesForExecution, testExecution) + val configuration = dockerService.prepareConfiguration(testExecution) testContainerId = dockerService.createContainers( testExecution.id!!, configuration @@ -111,7 +103,6 @@ class DockerServiceTest { // assertions Thread.sleep(2_500) // waiting for container to start val inspectContainerResponse = dockerClient.inspectContainerCmd(testContainerId).exec() - testImageId = inspectContainerResponse.imageId Assertions.assertTrue(inspectContainerResponse.state.running!!) { dockerClient.logContainerCmd(testContainerId) .withStdOut(true) @@ -137,9 +128,6 @@ class DockerServiceTest { if (::testContainerId.isInitialized) { dockerClient.removeContainerCmd(testContainerId).exec() } - if (::testImageId.isInitialized) { - dockerClient.removeImageCmd(testImageId).exec() - } mockServer.checkQueues() mockServer.cleanup() } diff --git a/save-orchestrator/src/test/resources/application.properties b/save-orchestrator/src/test/resources/application.properties index 93f82bed4c..7279428b2e 100644 --- a/save-orchestrator/src/test/resources/application.properties +++ b/save-orchestrator/src/test/resources/application.properties @@ -1,23 +1,23 @@ -orchestrator.backendUrl=http://backend:5800/internal -orchestrator.executionLogs=${user.home}/.save-cloud/test/executionLogs/ +orchestrator.backend-url=http://backend:5800/internal +orchestrator.execution-logs=${user.home}/.save-cloud/test/executionLogs/ orchestrator.test-resources.tmp-path=${user.home}/.save-cloud/test/tmp -orchestrator.shutdown.checksIntervalMillis=100 -orchestrator.shutdown.gracefulTimeoutSeconds=60 -orchestrator.shutdown.gracefulNumChecks=10 +orchestrator.shutdown.checks-interval-millis=100 +orchestrator.shutdown.graceful-timeout-seconds=60 +orchestrator.shutdown.graceful-num-checks=10 server.port = 5100 management.endpoints.web.exposure.include=health,info,prometheus # use IP from WSL on windows (`ip a | grep eth0`), because runsc can't be installed on windows #orchestrator.docker.host=tcp://172.25.71.25:2375 orchestrator.docker.host=unix:///var/run/docker.sock -orchestrator.docker.loggingDriver=json +orchestrator.docker.logging-driver=json orchestrator.docker.runtime=runsc orchestrator.docker.test-resources-volume-type=bind -orchestrator.agentsCount=1 -orchestrator.adjustResourceOwner=false -orchestrator.agentsHeartBeatTimeoutMillis=5000 -orchestrator.heartBeatInspectorInterval=3 -orchestrator.agentsStartTimeoutMillis=60000 -orchestrator.agentsStartCheckIntervalMillis=10000 +orchestrator.agents-count=1 +orchestrator.adjust-resource-owner=false +orchestrator.agents-heart-beat-timeout-millis=5000 +orchestrator.heart-beat-inspector-interval=3 +orchestrator.agents-start-timeout-millis=60000 +orchestrator.agents-start-check-interval-millis=10000 logging.level.com.github.dockerjava=DEBUG logging.level.com.saveourtool.save.testutils.LoggingQueueDispatcher=INFO logging.level.com.saveourtool.save.orchestrator=DEBUG