From fa1fe7168af4815799d96ce242f4889edf40efd3 Mon Sep 17 00:00:00 2001 From: Shruti Mantri Date: Sun, 22 Sep 2024 22:17:18 -0700 Subject: [PATCH 1/2] feat(docs): add examples for include/exclude in namespace files --- .../plugin/scripts/python/Commands.java | 6 ++- .../kestra/plugin/scripts/ruby/Commands.java | 4 +- .../kestra/plugin/scripts/shell/Commands.java | 52 +++++++++++++++++-- 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java b/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java index 4cfe540..e953f0b 100644 --- a/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java +++ b/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java @@ -28,7 +28,7 @@ @Example( full = true, title = """ - Execute a Python script in a Conda virtual environment. First, add the following script in the embedded VS Code editor and name it `etl_script.py`: + Execute a Python script in a Conda virtual environment. First, add the following script in the embedded Code Editor and name it `etl_script.py`: ```python import argparse @@ -42,7 +42,7 @@ print(result) ``` - Then, make sure to set the `enabled` flag of the `namespaceFiles` property to `true` to enable [namespace files](https://kestra.io/docs/developer-guide/namespace-files). + Then, make sure to set the `enabled` flag of the `namespaceFiles` property to `true` to enable [namespace files](https://kestra.io/docs/developer-guide/namespace-files). We `include` only the `etl_script.py` file as that is the only file we require from namespace files. This flow uses a `io.kestra.plugin.core.runner.Process` Task Runner and Conda virtual environment for process isolation and dependency management. However, note that, by default, Kestra runs tasks in a Docker container (i.e. a Docker task runner), and you can use the `taskRunner` property to customize many options, as well as `containerImage` to choose the Docker image to use. """, @@ -55,6 +55,8 @@ type: io.kestra.plugin.scripts.python.Commands namespaceFiles: enabled: true + include: + - etl_script.py taskRunner: type: io.kestra.plugin.core.runner.Process beforeCommands: diff --git a/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java b/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java index f9d57b5..8bbaf55 100644 --- a/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java +++ b/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java @@ -51,7 +51,7 @@ end ``` - In order to read that script from the [Namespace File](https://kestra.io/docs/developer-guide/namespace-files) called `main.rb`, you need to enable the `namespaceFiles` property. + In order to read that script from the [Namespace File](https://kestra.io/docs/developer-guide/namespace-files) called `main.rb`, you need to enable the `namespaceFiles` property. We include only `main.rb` as that is the only file we want from the `namespaceFiles`. Also, note how we use the `inputFiles` option to read additional files into the script's working directory. In this case, we read the `data.json` file, which contains the data that we want to convert to CSV. @@ -66,6 +66,8 @@ type: io.kestra.plugin.scripts.ruby.Commands namespaceFiles: enabled: true + include: + - main.rb inputFiles: data.json: | [ diff --git a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java index 783d68a..89307f6 100644 --- a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java +++ b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java @@ -56,6 +56,46 @@ - 'echo "The current execution is: {{ execution.id }}"' """ ), + @Example( + title = "Include only specific namespace files.", + full = true, + code = """ + id: include_files + namespace: company.team + + tasks: + - id: command + type: io.kestra.plugin.scripts.shell.Commands + description: "Only the included `namespaceFiles` get listed" + namespaceFiles: + enabled: true + include: + - test1.txt + - test2.yaml + commands: + - ls + """ + ), + @Example( + title = "Exclude specific namespace files.", + full = true, + code = """ + id: exclude_files + namespace: company.team + + tasks: + - id: command + type: io.kestra.plugin.scripts.shell.Commands + description: "All the `namespaceFiles` except those that are excluded get listed" + namespaceFiles: + enabled: true + exclude: + - test1.txt + - test2.yaml + commands: + - ls + """ + ), @Example( title = "Execute Shell commands that generate files accessible by other tasks and available for download in the UI's Output tab.", full = true, @@ -82,10 +122,14 @@ namespace: company.team tasks: - - id: commands - type: io.kestra.plugin.scripts.shell.Commands - commands: - - cat {{ outputs.previousTaskId.uri }} + - id: http_download + type: io.kestra.plugin.core.http.Download + uri: https://huggingface.co/datasets/kestra/datasets/raw/main/csv/products.csv + + - id: commands + type: io.kestra.plugin.scripts.shell.Commands + commands: + - cat {{ outputs.http_download.uri }} """ ), @Example( From a18a218a96304006d01469fcff28373646bf3507 Mon Sep 17 00:00:00 2001 From: Anna Geller Date: Mon, 23 Sep 2024 09:29:19 +0200 Subject: [PATCH 2/2] Update plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java --- .../src/main/java/io/kestra/plugin/scripts/shell/Commands.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java index 89307f6..0b3770b 100644 --- a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java +++ b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java @@ -86,7 +86,7 @@ tasks: - id: command type: io.kestra.plugin.scripts.shell.Commands - description: "All the `namespaceFiles` except those that are excluded get listed" + description: "All `namespaceFiles` except those that are excluded will be injected into the task's working directory" namespaceFiles: enabled: true exclude: