From a52d2a256662d4e5b1fafb6c04fc5548e807cae8 Mon Sep 17 00:00:00 2001 From: Ludovic DEHON Date: Tue, 14 Nov 2023 12:51:56 +0100 Subject: [PATCH] feat(script): reintroduce input & output files --- .../plugin/scripts/shell/ScriptTest.java | 57 ++++++++++++++++++- .../scripts/exec/AbstractExecScript.java | 15 +++-- .../exec/scripts/runners/CommandsWrapper.java | 23 +++++++- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/ScriptTest.java b/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/ScriptTest.java index 26cbcfb8..9a666232 100644 --- a/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/ScriptTest.java +++ b/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/ScriptTest.java @@ -7,7 +7,8 @@ import io.kestra.core.queues.QueueInterface; import io.kestra.core.runners.RunContext; import io.kestra.core.runners.RunContextFactory; -import io.kestra.core.utils.Rethrow; +import io.kestra.core.storages.StorageInterface; +import io.kestra.core.utils.IdUtils; import io.kestra.core.utils.TestsUtils; import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions; import io.kestra.plugin.scripts.exec.scripts.models.RunnerType; @@ -20,18 +21,27 @@ import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Stream; +import static io.kestra.core.utils.Rethrow.throwFunction; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.*; @MicronautTest class ScriptTest { @Inject - RunContextFactory runContextFactory; + private RunContextFactory runContextFactory; + + @Inject + private StorageInterface storageInterface; @Inject @Named(QueueFactoryInterface.WORKERTASKLOG_NAMED) @@ -132,4 +142,45 @@ void overwrite() throws Exception { assertThat(((Map>>)run.getVars().get("config")).get("auths").get("own.registry").get("username"), is("jane")); assertThat(run.getExitCode(), is(0)); } + + @Test + void inputOutputFiles() throws Exception { + Script bash = Script.builder() + .id("unit-test") + .type(Script.class.getName()) + .inputFiles(Map.of( + "test/application.yml", internalFiles("/test/" + IdUtils.create() + ".yml").toString() + )) + .docker(DockerOptions.builder() + .user("1000") + .image("ubuntu") + .build() + ) + .outputFiles(List.of("out/**")) + .script(""" + mkdir out + cat test/application.yml > out/bla.yml + """) + .build(); + + RunContext runContext = TestsUtils.mockRunContext(runContextFactory, bash, ImmutableMap.of()); + ScriptOutput run = bash.run(runContext); + + assertThat(run.getExitCode(), is(0)); + assertThat(run.getOutputFiles().get("out/bla.yml").toString(), startsWith("kestra://")); + assertThat( + new String(storageInterface.get(null, run.getOutputFiles().get("out/bla.yml")).readAllBytes()), + containsString("base-path: /tmp/unittest") + ); + } + + private URI internalFiles(String path) throws IOException, URISyntaxException { + var resource = ScriptTest.class.getClassLoader().getResource("application.yml"); + + return storageInterface.put( + null, + new URI(path), + new FileInputStream(Objects.requireNonNull(resource).getFile()) + ); + } } diff --git a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java index 05e6f7dd..180537f3 100644 --- a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java +++ b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java @@ -2,10 +2,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.annotations.PluginProperty; -import io.kestra.core.models.tasks.NamespaceFiles; -import io.kestra.core.models.tasks.NamespaceFilesInterface; -import io.kestra.core.models.tasks.RunnableTask; -import io.kestra.core.models.tasks.Task; +import io.kestra.core.models.tasks.*; import io.kestra.core.runners.RunContext; import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions; import io.kestra.plugin.scripts.exec.scripts.models.RunnerType; @@ -25,7 +22,7 @@ @EqualsAndHashCode @Getter @NoArgsConstructor -public abstract class AbstractExecScript extends Task implements RunnableTask, NamespaceFilesInterface { +public abstract class AbstractExecScript extends Task implements RunnableTask, NamespaceFilesInterface, InputFilesInterface, OutputFilesInterface { @Builder.Default @Schema( title = "Runner to use" @@ -69,6 +66,10 @@ public abstract class AbstractExecScript extends Task implements RunnableTask outputFiles; + abstract public DockerOptions getDocker(); /** @@ -95,6 +96,8 @@ protected CommandsWrapper commands(RunContext runContext) throws IllegalVariable .withWarningOnStdErr(this.getWarningOnStdErr()) .withRunnerType(this.getRunner()) .withDockerOptions(this.injectDefaults(getDocker())) - .withNamespaceFiles(namespaceFiles); + .withNamespaceFiles(namespaceFiles) + .withInputFiles(this.inputFiles) + .withOutputFiles(this.outputFiles); } } diff --git a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java index 84f521ed..ffa4d33e 100644 --- a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java +++ b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java @@ -2,6 +2,7 @@ import io.kestra.core.exceptions.IllegalVariableEvaluationException; import io.kestra.core.models.tasks.NamespaceFiles; +import io.kestra.core.runners.FilesService; import io.kestra.core.runners.NamespaceFilesService; import io.kestra.core.runners.RunContext; import io.kestra.core.utils.IdUtils; @@ -54,6 +55,12 @@ public class CommandsWrapper { @With private NamespaceFiles namespaceFiles; + @With + private Object inputFiles; + + @With + private List outputFiles; + public CommandsWrapper(RunContext runContext) { this.runContext = runContext; @@ -82,7 +89,9 @@ public CommandsWrapper withCommands(List commands) throws IOException, I runnerType, dockerOptions, warningOnStdErr, - namespaceFiles + namespaceFiles, + inputFiles, + outputFiles ); } @@ -106,7 +115,9 @@ public CommandsWrapper withEnv(Map envs) throws IllegalVariableE runnerType, dockerOptions, warningOnStdErr, - namespaceFiles + namespaceFiles, + inputFiles, + outputFiles ); } @@ -140,6 +151,10 @@ public ScriptOutput run() throws Exception { ); } + if (this.inputFiles != null) { + FilesService.inputFiles(runContext, this.inputFiles); + } + if (runnerType.equals(RunnerType.DOCKER)) { runnerResult = new DockerScriptRunner(runContext.getApplicationContext()).run(this, this.dockerOptions); } else { @@ -148,6 +163,10 @@ public ScriptOutput run() throws Exception { Map outputFiles = ScriptService.uploadOutputFiles(runContext, outputDirectory); + if (this.outputFiles != null) { + outputFiles.putAll(FilesService.outputFiles(runContext, this.outputFiles)); + } + return ScriptOutput.builder() .exitCode(runnerResult.getExitCode()) .stdOutLineCount(runnerResult.getLogConsumer().getStdOutCount())