feat(script): reintroduce input & output files

kestra-io · Nov 14, 2023 · a52d2a2 · a52d2a2
1 parent 58de46d
commit a52d2a2
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 11 deletions.
diff --git a/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/ScriptTest.java b/plugin-script-shell/src/test/java/io/kestra/plugin/scripts/shell/ScriptTest.java
@@ -7,7 +7,8 @@
 import io.kestra.core.queues.QueueInterface;
 import io.kestra.core.runners.RunContext;
 import io.kestra.core.runners.RunContextFactory;
-import io.kestra.core.utils.Rethrow;
+import io.kestra.core.storages.StorageInterface;
+import io.kestra.core.utils.IdUtils;
 import io.kestra.core.utils.TestsUtils;
 import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions;
 import io.kestra.plugin.scripts.exec.scripts.models.RunnerType;
@@ -20,18 +21,27 @@
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Stream;
 
+import static io.kestra.core.utils.Rethrow.throwFunction;
 import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.*;
 
 @MicronautTest
 class ScriptTest {
     @Inject
-    RunContextFactory runContextFactory;
+    private RunContextFactory runContextFactory;
+
+    @Inject
+    private StorageInterface storageInterface;
 
     @Inject
     @Named(QueueFactoryInterface.WORKERTASKLOG_NAMED)
@@ -132,4 +142,45 @@ void overwrite() throws Exception {
         assertThat(((Map<String, Map<String, Map<String, Object>>>)run.getVars().get("config")).get("auths").get("own.registry").get("username"), is("jane"));
         assertThat(run.getExitCode(), is(0));
     }
+
+    @Test
+    void inputOutputFiles() throws Exception {
+        Script bash = Script.builder()
+            .id("unit-test")
+            .type(Script.class.getName())
+            .inputFiles(Map.of(
+                "test/application.yml", internalFiles("/test/" + IdUtils.create() + ".yml").toString()
+            ))
+            .docker(DockerOptions.builder()
+                .user("1000")
+                .image("ubuntu")
+                .build()
+            )
+            .outputFiles(List.of("out/**"))
+            .script("""
+                mkdir out
+                cat test/application.yml > out/bla.yml
+            """)
+            .build();
+
+        RunContext runContext = TestsUtils.mockRunContext(runContextFactory, bash, ImmutableMap.of());
+        ScriptOutput run = bash.run(runContext);
+
+        assertThat(run.getExitCode(), is(0));
+        assertThat(run.getOutputFiles().get("out/bla.yml").toString(), startsWith("kestra://"));
+        assertThat(
+            new String(storageInterface.get(null, run.getOutputFiles().get("out/bla.yml")).readAllBytes()),
+            containsString("base-path: /tmp/unittest")
+        );
+    }
+
+    private URI internalFiles(String path) throws IOException, URISyntaxException {
+        var resource = ScriptTest.class.getClassLoader().getResource("application.yml");
+
+        return storageInterface.put(
+            null,
+            new URI(path),
+            new FileInputStream(Objects.requireNonNull(resource).getFile())
+        );
+    }
 }
diff --git a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java
@@ -2,10 +2,7 @@
 
 import io.kestra.core.exceptions.IllegalVariableEvaluationException;
 import io.kestra.core.models.annotations.PluginProperty;
-import io.kestra.core.models.tasks.NamespaceFiles;
-import io.kestra.core.models.tasks.NamespaceFilesInterface;
-import io.kestra.core.models.tasks.RunnableTask;
-import io.kestra.core.models.tasks.Task;
+import io.kestra.core.models.tasks.*;
 import io.kestra.core.runners.RunContext;
 import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions;
 import io.kestra.plugin.scripts.exec.scripts.models.RunnerType;
@@ -25,7 +22,7 @@
 @EqualsAndHashCode
 @Getter
 @NoArgsConstructor
-public abstract class AbstractExecScript extends Task implements RunnableTask<ScriptOutput>, NamespaceFilesInterface {
+public abstract class AbstractExecScript extends Task implements RunnableTask<ScriptOutput>, NamespaceFilesInterface, InputFilesInterface, OutputFilesInterface {
     @Builder.Default
     @Schema(
         title = "Runner to use"
@@ -69,6 +66,10 @@ public abstract class AbstractExecScript extends Task implements RunnableTask<Sc
 
     private NamespaceFiles namespaceFiles;
 
+    private Object inputFiles;
+
+    private List<String> outputFiles;
+
     abstract public DockerOptions getDocker();
 
     /**
@@ -95,6 +96,8 @@ protected CommandsWrapper commands(RunContext runContext) throws IllegalVariable
             .withWarningOnStdErr(this.getWarningOnStdErr())
             .withRunnerType(this.getRunner())
             .withDockerOptions(this.injectDefaults(getDocker()))
-            .withNamespaceFiles(namespaceFiles);
+            .withNamespaceFiles(namespaceFiles)
+            .withInputFiles(this.inputFiles)
+            .withOutputFiles(this.outputFiles);
     }
 }
diff --git a/...n-script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java b/...n-script/src/main/java/io/kestra/plugin/scripts/exec/scripts/runners/CommandsWrapper.java
@@ -2,6 +2,7 @@
 
 import io.kestra.core.exceptions.IllegalVariableEvaluationException;
 import io.kestra.core.models.tasks.NamespaceFiles;
+import io.kestra.core.runners.FilesService;
 import io.kestra.core.runners.NamespaceFilesService;
 import io.kestra.core.runners.RunContext;
 import io.kestra.core.utils.IdUtils;
@@ -54,6 +55,12 @@ public class CommandsWrapper {
     @With
     private NamespaceFiles namespaceFiles;
 
+    @With
+    private Object inputFiles;
+
+    @With
+    private List<String> outputFiles;
+
     public CommandsWrapper(RunContext runContext) {
         this.runContext = runContext;
 
@@ -82,7 +89,9 @@ public CommandsWrapper withCommands(List<String> commands) throws IOException, I
             runnerType,
             dockerOptions,
             warningOnStdErr,
-            namespaceFiles
+            namespaceFiles,
+            inputFiles,
+            outputFiles
         );
     }
 
@@ -106,7 +115,9 @@ public CommandsWrapper withEnv(Map<String, String> envs) throws IllegalVariableE
             runnerType,
             dockerOptions,
             warningOnStdErr,
-            namespaceFiles
+            namespaceFiles,
+            inputFiles,
+            outputFiles
         );
     }
 
@@ -140,6 +151,10 @@ public ScriptOutput run() throws Exception {
             );
         }
 
+        if (this.inputFiles != null) {
+            FilesService.inputFiles(runContext, this.inputFiles);
+        }
+
         if (runnerType.equals(RunnerType.DOCKER)) {
             runnerResult = new DockerScriptRunner(runContext.getApplicationContext()).run(this, this.dockerOptions);
         } else {
@@ -148,6 +163,10 @@ public ScriptOutput run() throws Exception {
 
         Map<String, URI> outputFiles = ScriptService.uploadOutputFiles(runContext, outputDirectory);
 
+        if (this.outputFiles != null) {
+            outputFiles.putAll(FilesService.outputFiles(runContext, this.outputFiles));
+        }
+
         return ScriptOutput.builder()
             .exitCode(runnerResult.getExitCode())
             .stdOutLineCount(runnerResult.getLogConsumer().getStdOutCount())