diff --git a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java index c1f6c156..6943c078 100644 --- a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java +++ b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java @@ -21,32 +21,30 @@ @Getter @NoArgsConstructor @Schema( - title = "Execute one or more Julia commands from the Command Line Interface." + title = "Execute Julia scripts from the Command Line Interface." ) @Plugin(examples = { @Example( full = true, - title = "Install package, create a julia script and execute it", + title = "Create a Julia script, install required packages and execute it. Note that instead of defining the script inline, you could create the Julia script in the embedded VS Code editor and point to its location by path. If you do so, make sure to enable namespace files by setting the `enabled` flag of the `namespaceFiles` property to `true`.", code = """ - id: "local-files" - namespace: "io.kestra.tests" - + id: "script" + namespace: "dev" tasks: - - id: workingDir - type: io.kestra.core.tasks.flows.WorkingDirectory - tasks: - - id: inputFiles - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - main.js: | - const colors = require("colors"); - console.log(colors.red("Hello")); - - id: bash - type: io.kestra.plugin.scripts.julia.Commands - beforeCommands: - - npm install colors - commands: - - julia main.jl + - id: bash + type: io.kestra.plugin.scripts.julia.Commands + warningOnStdErr: false + inputFiles: + main.jl: | + using DataFrames, CSV + df = DataFrame(Name = ["Alice", "Bob", "Charlie"], Age = [25, 30, 35]) + CSV.write("output.csv", df) + outputFiles: + - output.csv + beforeCommands: + - julia -e 'using Pkg; Pkg.add("DataFrames"); Pkg.add("CSV")' + commands: + - julia main.jl """ ) }) diff --git a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java index e71c395b..e73c9664 100644 --- a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java +++ b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java @@ -29,12 +29,24 @@ ) @Plugin(examples = { @Example( - title = "Create a julia script and execute it", - code = { - "script: |", - " const colors = require(\"colors\");", - " console.log(colors.red(\"Hello\"));", - } + full = true, + title = "Create a Julia script, install required packages and execute it. Note that instead of defining the script inline, you could create the Julia script in the embedded VS Code editor and read its content using the `{{ read('your_script.jl') }}` function.", + code = """ + id: "script" + namespace: "dev" + tasks: + - id: bash + type: io.kestra.plugin.scripts.julia.Script + warningOnStdErr: false + script: | + using DataFrames, CSV + df = DataFrame(Name = ["Alice", "Bob", "Charlie"], Age = [25, 30, 35]) + CSV.write("output.csv", df) + outputFiles: + - output.csv + beforeCommands: + - julia -e 'using Pkg; Pkg.add("DataFrames"); Pkg.add("CSV")' + """ ) }) public class Script extends AbstractExecScript { @@ -49,7 +61,7 @@ public class Script extends AbstractExecScript { protected DockerOptions docker = DockerOptions.builder().build(); @Schema( - title = "The inline script content. This property is intended for the script file's content as a (multiline) string, not a path to a file. To run a command from a file such as `bash myscript.sh` or `python myscript.py`, use the `Commands` task instead." + title = "The inline script content. This property is intended for the script file's content as a (multiline) string, not a path to a file. To run a command such as `julia myscript.jl`, use the `Commands` task instead." ) @PluginProperty(dynamic = true) @NotNull @@ -81,7 +93,6 @@ public ScriptOutput run(RunContext runContext) throws Exception { ); return commands - .addEnv(Map.of("PYTHONUNBUFFERED", "true")) .withCommands(commandsArgs) .run(); } diff --git a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java index afa3523a..f8e2a5b5 100644 --- a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java +++ b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java @@ -21,32 +21,26 @@ @Getter @NoArgsConstructor @Schema( - title = "Execute one or more Node commands from the Command Line Interface." + title = "Execute one or more Node commands from the Command Line Interface. Note that instead of adding the script using the `inputFiles` property, you could also add the script from the embedded VS Code editor and point to its location by path. If you do so, make sure to enable namespace files by setting the `enabled` flag of the `namespaceFiles` property to `true`." ) @Plugin(examples = { @Example( full = true, - title = "Install package, create a node script and execute it", + title = "Install required npm packages, create a Node script and execute it.", code = """ - id: "local-files" - namespace: "io.kestra.tests" - + id: node + namespace: dev tasks: - - id: workingDir - type: io.kestra.core.tasks.flows.WorkingDirectory - tasks: - - id: inputFiles - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - main.js: | - const colors = require("colors"); - console.log(colors.red("Hello")); - - id: bash - type: io.kestra.plugin.scripts.node.Commands - beforeCommands: - - npm install colors - commands: - - node main.js + - id: node_script + type: io.kestra.plugin.scripts.node.Commands + inputFiles: + main.js: | + const colors = require("colors"); + console.log(colors.red("Hello")); + beforeCommands: + - npm install colors + commands: + - node main.js """ ) }) diff --git a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java index fa32e493..ac5ccb50 100644 --- a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java +++ b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java @@ -41,7 +41,9 @@ @Example( full = true, title = """ - If you want to generate files in your script to make them available for download and use in downstream tasks, you can leverage the `{{outputDir}}` variable. Files stored in that directory will be persisted in Kestra's internal storage. To access this output in downstream tasks, use the syntax `{{outputs.yourTaskId.outputFiles['yourFileName.fileExtension']}}`. + If you want to generate files in your script to make them available for download and use in downstream tasks, you can leverage the `{{outputDir}}` variable. Files stored in that directory will be persisted in Kestra's internal storage. To access this output in downstream tasks, use the syntax `{{outputs.yourTaskId.outputFiles['yourFileName.fileExtension']}}`. + + Alternatively, instead of the `{{outputDir}}` variable, you could use the `outputFiles` property to output files from your script. You can access those files in downstream tasks using the same syntax `{{outputs.yourTaskId.outputFiles['yourFileName.fileExtension']}}`, and you can download the files from the UI's Output tab. """, code = """ id: nodeJS diff --git a/plugin-script-powershell/src/main/java/io/kestra/plugin/scripts/powershell/Commands.java b/plugin-script-powershell/src/main/java/io/kestra/plugin/scripts/powershell/Commands.java index 17b44587..98ccf746 100644 --- a/plugin-script-powershell/src/main/java/io/kestra/plugin/scripts/powershell/Commands.java +++ b/plugin-script-powershell/src/main/java/io/kestra/plugin/scripts/powershell/Commands.java @@ -23,29 +23,23 @@ @Getter @NoArgsConstructor @Schema( - title = "Execute one or more PowerShell commands." + title = "Execute one or more PowerShell commands. Note that instead of adding the script using the `inputFiles` property, you could also add the script from the embedded VS Code editor and point to its location by path. If you do so, make sure to enable namespace files by setting the `enabled` flag of the `namespaceFiles` property to `true`." ) @Plugin(examples = { @Example( full = true, - title = "Create a PowerShell script and execute it", + title = "Create a PowerShell script and execute it.", code = """ - id: "local-files" - namespace: "io.kestra.tests" - + id: powershell + namespace: dev tasks: - - id: workingDir - type: io.kestra.core.tasks.flows.WorkingDirectory - tasks: - - id: inputFiles - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - main.ps1: | - Get-ChildItem | Format-List - - id: bash - type: io.kestra.plugin.scripts.powershell.Commands - commands: - - pwsh main.ps1 + - id: powershell_script + type: io.kestra.plugin.scripts.powershell.Commands + inputFiles: + main.ps1: | + Get-ChildItem | Format-List + commands: + - pwsh main.ps1 """ ) }) diff --git a/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java b/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java index beed6365..9548da6f 100644 --- a/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java +++ b/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Commands.java @@ -26,6 +26,44 @@ title = "Execute one or more Python scripts from a Command Line Interface." ) @Plugin(examples = { + @Example( + full = true, + title = """ + Execute a Python script in a Conda virtual environment. First, add the following script in the embedded VS Code editor and name it `etl_script.py`: + + ```python + import argparse + + parser = argparse.ArgumentParser() + + parser.add_argument("--num", type=int, default=42, help="Enter an integer") + + args = parser.parse_args() + result = args.num * 2 + print(result) + ``` + + Then, make sure to set the `enabled` flag of the `namespaceFiles` property to `true` to enable [namespace files](https://kestra.io/docs/developer-guide/namespace-files). + ``` + + This flow uses a `PROCESS` runner and Conda virtual environment for process isolation and dependency management. However, note that, by default, Kestra runs tasks in a Docker container (i.e. a `DOCKER` runner), and you can use the `docker` property to customize many options, such as the Docker image to use. + """, + code = """ +id: python_venv +namespace: dev + +tasks: + - id: hello + type: io.kestra.plugin.scripts.python.Commands + namespaceFiles: + enabled: true + runner: PROCESS + beforeCommands: + - conda activate myCondaEnv + commands: + - python etl_script.py + """ + ), @Example( full = true, title = "Execute a Python script from Git in a Docker container and output a file", @@ -52,10 +90,7 @@ commands: - python scripts/etl_script.py - python scripts/generate_orders.py - - - id: outputFile - type: io.kestra.core.tasks.storages.LocalFiles - outputs: + outputFiles: - orders.csv - id: loadCsvToS3 @@ -68,23 +103,6 @@ from: "{{outputs.outputFile.uris['orders.csv']}}" """ ), - @Example( - full = true, - title = "Execute a Python script in a Conda virtual environment", - code = """ -id: localPythonScript -namespace: dev - -tasks: - - id: hello - type: io.kestra.plugin.scripts.python.Commands - runner: PROCESS - beforeCommands: - - conda activate myCondaEnv - commands: - - python /Users/you/scripts/etl_script.py - """ - ), @Example( full = true, title = "Execute a Python script on a remote worker with a GPU", @@ -119,23 +137,17 @@ url: https://github.com/kestra-io/examples branch: main - - id: local - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - data.csv: "{{ trigger.objects | jq('.[].uri') | first }}" - - id: python type: io.kestra.plugin.scripts.python.Commands + inputFiles: + data.csv: "{{ trigger.objects | jq('.[].uri') | first }}" description: this script reads a file `data.csv` from S3 trigger docker: image: ghcr.io/kestra-io/pydata:latest warningOnStdErr: false commands: - python scripts/clean_messy_dataset.py - - - id: output - type: io.kestra.core.tasks.storages.LocalFiles - outputs: + outputFiles: - "*.csv" - "*.parquet" @@ -188,9 +200,10 @@ } } } + - id: output type: io.kestra.core.tasks.storages.LocalFiles - outputs: + outputFiles: - "*.csv" - "*.parquet" """ @@ -199,31 +212,25 @@ full = true, title = "Create a python script and execute it in a virtual environment", code = """ - id: "local-files" - namespace: "io.kestra.tests" - + id: "script_in_venv" + namespace: "dev" tasks: - - id: workingDir - type: io.kestra.core.tasks.flows.WorkingDirectory - tasks: - - id: inputFiles - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - main.py: | - import requests - from kestra import Kestra + - id: bash + type: io.kestra.plugin.scripts.python.Commands + inputFiles: + main.py: | + import requests + from kestra import Kestra - response = requests.get('https://google.com') - print(response.status_code) - Kestra.outputs({'status': response.status_code, 'text': response.text}) - - id: bash - type: io.kestra.plugin.scripts.python.Commands - beforeCommands: - - python -m venv venv - - . venv/bin/activate - - pip install requests kestra > /dev/null - commands: - - python main.py + response = requests.get('https://google.com') + print(response.status_code) + Kestra.outputs({'status': response.status_code, 'text': response.text}) + beforeCommands: + - python -m venv venv + - . venv/bin/activate + - pip install requests kestra > /dev/null + commands: + - python main.py """ ) }) diff --git a/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Script.java b/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Script.java index 2da06c5d..afecb62e 100644 --- a/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Script.java +++ b/plugin-script-python/src/main/java/io/kestra/plugin/scripts/python/Script.java @@ -57,7 +57,7 @@ code = { "script: |", " f = open(\"{{outputDir}}/myfile.txt\", \"a\")", - " f.write(\"I can output files from my script!\")", + " f.write(\"Hello from a Kestra task!\")", " f.close()" } ), diff --git a/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Commands.java b/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Commands.java index cba1b279..6e84892c 100644 --- a/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Commands.java +++ b/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Commands.java @@ -22,34 +22,28 @@ @Getter @NoArgsConstructor @Schema( - title = "Execute R from the Command Line Interface." + title = "Execute R scripts from the Command Line Interface." ) @Plugin(examples = { @Example( full = true, - title = "Create a R script, install a package and execute it", + title = "Create an R script, install required packages and execute it. Note that instead of defining the script inline, you could create the script as a dedicated R script in the embedded VS Code editor and point to its location by path. If you do so, make sure to enable namespace files by setting the `enabled` flag of the `namespaceFiles` property to `true`.", code = """ - id: "local-files" - namespace: "io.kestra.tests" - + id: "script" + namespace: "dev" tasks: - - id: workingDir - type: io.kestra.core.tasks.flows.WorkingDirectory - tasks: - - id: inputFiles - type: io.kestra.core.tasks.storages.LocalFiles - inputs: - main.R: | - library(lubridate) - ymd("20100604"); - mdy("06-04-2011"); - dmy("04/06/2012") - - id: bash - type: io.kestra.plugin.scripts.powershell.Commands - beforeCommands: - - Rscript -e 'install.packages("lubridate")' - commands: - - Rscript main.R + - id: bash + type: io.kestra.plugin.scripts.r.Commands + inputFiles: + main.R: | + library(lubridate) + ymd("20100604"); + mdy("06-04-2011"); + dmy("04/06/2012") + beforeCommands: + - Rscript -e 'install.packages("lubridate")' + commands: + - Rscript main.R """ ) }) diff --git a/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Script.java b/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Script.java index 451920f3..e635fec2 100644 --- a/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Script.java +++ b/plugin-script-r/src/main/java/io/kestra/plugin/scripts/r/Script.java @@ -29,7 +29,7 @@ @Plugin( examples = { @Example( - title = "Install a package and execute a R script", + title = "Install a package and execute an R script", code = { "script: |", " library(lubridate)", @@ -43,43 +43,58 @@ @Example( full = true, title = """ - If you want to generate files in your script to make them available for download and use in downstream tasks, you can leverage the `{{outputDir}}` variable. Files stored in that directory will be persisted in Kestra's internal storage. To access this output in downstream tasks, use the syntax `{{outputs.yourTaskId.outputFiles['yourFileName.fileExtension']}}`. + Add an R script in the embedded VS Code editor, install required packages and execute it. + + Here is an example R script that you can add in the embedded VS Code editor. You can name the script file `main.R`: + + ```r + library(dplyr) + library(arrow) + + data(mtcars) # load mtcars data + print(head(mtcars)) + + final <- mtcars %>% + summarise( + avg_mpg = mean(mpg), + avg_disp = mean(disp), + avg_hp = mean(hp), + avg_drat = mean(drat), + avg_wt = mean(wt), + avg_qsec = mean(qsec), + avg_vs = mean(vs), + avg_am = mean(am), + avg_gear = mean(gear), + avg_carb = mean(carb) + ) + final %>% print() + write.csv(final, "final.csv") + + mtcars_clean <- na.omit(mtcars) # this line removes rows with NA values + write_parquet(mtcars_clean, "mtcars_clean.parquet") + ``` + + Note that tasks in Kestra are stateless. Therefore, the files generated by a task, such as the CSV and Parquet files in the example above, are not persisted in Kestra's internal storage, unless you explicitly tell Kestra to do so. Make sure to add the `outputFiles` property to your task as shown below to persist the generated Parquet file (or any other file) in Kestra's internal storage and make them visible in the **Outputs** tab. + + To access this output in downstream tasks, use the syntax `{{outputs.yourTaskId.outputFiles['yourFileName.fileExtension']}}`. Alternatively, you can wrap your tasks that need to pass data between each other in a `WorkingDirectory` task — this way, those tasks will share the same working directory and will be able to access the same files. + + Note how we use the `read` function to read the content of the R script stored as a [Namespace File](https://kestra.io/docs/developer-guide/namespace-files). + + Finally, note that the `docker` property is optional. If you don't specify it, Kestra will use the default R image. If you want to use a different image, you can specify it in the `docker` property as shown below. """, code = """ id: rCars namespace: dev - tasks: - id: r type: io.kestra.plugin.scripts.r.Script warningOnStdErr: false docker: image: ghcr.io/kestra-io/rdata:latest - script: | - library(dplyr) - library(arrow) - - data(mtcars) # Load mtcars data - print(head(mtcars)) - - final <- mtcars %>% - summarise( - avg_mpg = mean(mpg), - avg_disp = mean(disp), - avg_hp = mean(hp), - avg_drat = mean(drat), - avg_wt = mean(wt), - avg_qsec = mean(qsec), - avg_vs = mean(vs), - avg_am = mean(am), - avg_gear = mean(gear), - avg_carb = mean(carb) - ) - final %>% print() - write.csv(final, "{{outputDir}}/final.csv") - - mtcars_clean <- na.omit(mtcars) # remove rows with NA values - write_parquet(mtcars_clean, "{{outputDir}}/mtcars_clean.parquet") + script: "{{ read('main.R') }} + outputFiles: + - "*.csv" + - "*.parquet" """ ) } @@ -96,7 +111,7 @@ public class Script extends AbstractExecScript { protected DockerOptions docker = DockerOptions.builder().build(); @Schema( - title = "The inline script content. This property is intended for the script file's content as a (multiline) string, not a path to a file. To run a command from a file such as `bash myscript.sh` or `python myscript.py`, use the `Commands` task instead." + title = "The inline script content. This property is intended for the script file's content as a (multiline) string, not a path to a file. To run a command from a file such as `Rscript main.R` or `python main.py`, use the corresponding `Commands` task for a given language instead." ) @PluginProperty(dynamic = true) @NotNull diff --git a/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java b/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java new file mode 100644 index 00000000..bd156d82 --- /dev/null +++ b/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Commands.java @@ -0,0 +1,125 @@ +package io.kestra.plugin.scripts.ruby; + +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.scripts.exec.AbstractExecScript; +import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions; +import io.kestra.plugin.scripts.exec.scripts.models.ScriptOutput; +import io.kestra.plugin.scripts.exec.scripts.services.ScriptService; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.*; +import lombok.experimental.SuperBuilder; + +import java.util.List; +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Execute a Ruby script from the Command Line Interface." +) +@Plugin(examples = { + @Example( + full = true, + title = """ + Create a Ruby script and execute it. The easiest way to create a Ruby script is to use the embedded VS Code editor. Create a file named `main.rb` and paste the following code: + + ```ruby + require 'csv' + require 'json' + + file = File.read('data.json') + data_hash = JSON.parse(file) + + # Extract headers + headers = data_hash.first.keys + + # Convert hashes to arrays + data = data_hash.map(&:values) + + # Prepend headers to data + data.unshift(headers) + + # Create and write data to CSV file + CSV.open('output.csv', 'wb') do |csv| + data.each { |row| csv << row } + end + ``` + + In order to read that script from the [Namespace File](https://kestra.io/docs/developer-guide/namespace-files) called `main.rb`, you need to enable the `namespaceFiles` property. + + Also, note how we use the `inputFiles` option to read additional files into the script's working directory. In this case, we read the `data.json` file, which contains the data that we want to convert to CSV. + + Finally, we use the `outputFiles` option to specify that we want to output the `output.csv` file that is generated by the script. This allows us to access the file in the UI's Output tab and download it, or pass it to other tasks. + """, + code = """ + id: generate_csv + namespace: dev + tasks: + - id: bash + type: io.kestra.plugin.scripts.ruby.Commands + namespaceFiles: + enabled: true + inputFiles: + data.json: | + [ + {"Name": "Alice", "Age": 30, "City": "New York"}, + {"Name": "Bob", "Age": 22, "City": "Los Angeles"}, + {"Name": "Charlie", "Age": 35, "City": "Chicago"} + ] + beforeCommands: + - ruby -v + commands: + - ruby main.rb + outputFiles: + - "*.csv" + """ + ) +}) +public class Commands extends AbstractExecScript { + private static final String DEFAULT_IMAGE = "ruby"; + + @Schema( + title = "Docker options when using the `DOCKER` runner", + defaultValue = "{image=" + DEFAULT_IMAGE + ", pullPolicy=ALWAYS}" + ) + @PluginProperty + @Builder.Default + protected DockerOptions docker = DockerOptions.builder().build(); + + @Schema( + title = "The commands to run" + ) + @PluginProperty(dynamic = true) + @NotEmpty + protected List commands; + + @Override + protected DockerOptions injectDefaults(DockerOptions original) { + var builder = original.toBuilder(); + if (original.getImage() == null) { + builder.image(DEFAULT_IMAGE); + } + + return builder.build(); + } + + @Override + public ScriptOutput run(RunContext runContext) throws Exception { + List commandsArgs = ScriptService.scriptCommands( + this.interpreter, + this.beforeCommands, + this.commands + ); + + return this.commands(runContext) + .withCommands(commandsArgs) + .run(); + } +} diff --git a/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Script.java b/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Script.java new file mode 100644 index 00000000..29f7e5de --- /dev/null +++ b/plugin-script-ruby/src/main/java/io/kestra/plugin/scripts/ruby/Script.java @@ -0,0 +1,133 @@ +package io.kestra.plugin.scripts.ruby; + +import io.kestra.core.models.annotations.Example; +import io.kestra.core.models.annotations.Plugin; +import io.kestra.core.models.annotations.PluginProperty; +import io.kestra.core.runners.RunContext; +import io.kestra.plugin.scripts.exec.AbstractExecScript; +import io.kestra.plugin.scripts.exec.scripts.models.DockerOptions; +import io.kestra.plugin.scripts.exec.scripts.models.ScriptOutput; +import io.kestra.plugin.scripts.exec.scripts.runners.CommandsWrapper; +import io.kestra.plugin.scripts.exec.scripts.services.ScriptService; +import io.swagger.v3.oas.annotations.media.Schema; +import lombok.*; +import lombok.experimental.SuperBuilder; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; +import javax.validation.constraints.NotNull; + +@SuperBuilder +@ToString +@EqualsAndHashCode +@Getter +@NoArgsConstructor +@Schema( + title = "Execute a Ruby script." +) +@Plugin( + examples = { + @Example( + full = true, + title = """ + Create a Ruby script and execute it. The easiest way to create a Ruby script is to use the embedded VS Code editor. Create a file named `main.rb` and paste the following code: + + ```ruby + require 'csv' + require 'json' + + file = File.read('data.json') + data_hash = JSON.parse(file) + + # Extract headers + headers = data_hash.first.keys + + # Convert hashes to arrays + data = data_hash.map(&:values) + + # Prepend headers to data + data.unshift(headers) + + # Create and write data to CSV file + CSV.open('output.csv', 'wb') do |csv| + data.each { |row| csv << row } + end + ``` + + In order to read that script from the [Namespace File](https://kestra.io/docs/developer-guide/namespace-files) called `main.rb`, you can leverage the `{{ read('main.rb') }}` function. + + Also, note how we use the `inputFiles` option to read additional files into the script's working directory. In this case, we read the `data.json` file, which contains the data that we want to convert to CSV. + + Finally, we use the `outputFiles` option to specify that we want to output the `output.csv` file that is generated by the script. This allows us to access the file in the UI's Output tab and download it, or pass it to other tasks. + """, + code = """ + id: generate_csv + namespace: dev + tasks: + - id: bash + type: io.kestra.plugin.scripts.ruby.Script + inputFiles: + data.json: | + [ + {"Name": "Alice", "Age": 30, "City": "New York"}, + {"Name": "Bob", "Age": 22, "City": "Los Angeles"}, + {"Name": "Charlie", "Age": 35, "City": "Chicago"} + ] + beforeCommands: + - ruby -v + script: "{{ read('main.rb') }}" + outputFiles: + - "*.csv" + """ + ) + } +) +public class Script extends AbstractExecScript { + private static final String DEFAULT_IMAGE = "ruby"; + + @Schema( + title = "Docker options when using the `DOCKER` runner", + defaultValue = "{image=" + DEFAULT_IMAGE + ", pullPolicy=ALWAYS}" + ) + @PluginProperty + @Builder.Default + protected DockerOptions docker = DockerOptions.builder().build(); + + @Schema( + title = "The inline script content. This property is intended for the script file's content as a (multiline) string, not a path to a file. To run a command from a file such as `bash myscript.sh` or `python myscript.py`, use the `Commands` task instead." + ) + @PluginProperty(dynamic = true) + @NotNull + protected String script; + + @Override + protected DockerOptions injectDefaults(DockerOptions original) { + var builder = original.toBuilder(); + if (original.getImage() == null) { + builder.image(DEFAULT_IMAGE); + } + + return builder.build(); + } + + @Override + public ScriptOutput run(RunContext runContext) throws Exception { + CommandsWrapper commands = this.commands(runContext); + + Path path = runContext.tempFile( + ScriptService.replaceInternalStorage(runContext, runContext.render(this.script, commands.getAdditionalVars())).getBytes(StandardCharsets.UTF_8), + ".rb" + ); + + List commandsArgs = ScriptService.scriptCommands( + this.interpreter, + this.beforeCommands, + String.join(" ", "ruby", path.toAbsolutePath().toString()) + ); + + return commands + .withCommands(commandsArgs) + .run(); + } +} diff --git a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java index b4718f05..7bb581d9 100644 --- a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java +++ b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Commands.java @@ -30,34 +30,28 @@ full = true, title = "Execute ETL in Rust in a Docker container and output CSV files generated as a result of the script.", code = """ -id: rustFlow -namespace: dev -tasks: - - id: wdir - type: io.kestra.core.tasks.flows.WorkingDirectory - tasks: - - id: rust - type: io.kestra.plugin.scripts.shell.Commands - commands: - - etl - docker: - image: ghcr.io/kestra-io/rust:latest - - - id: downloadFiles - type: io.kestra.core.tasks.storages.LocalFiles - outputs: - - "*.csv" + id: rust_flow + namespace: dev + tasks: + - id: rust + type: io.kestra.plugin.scripts.shell.Commands + commands: + - etl + docker: + image: ghcr.io/kestra-io/rust:latest + outputFiles: + - "*.csv" """ ), @Example( - title = "Single shell command", + title = "Execute a single Shell command", code = { "commands:", - "- 'echo \"The current execution is : {{ execution.id }}\"'" + "- 'echo \"The current execution is: {{ execution.id }}\"'" } ), @Example( - title = "Shell command that generate file in storage accessible through outputs", + title = "Execute Shell commands that generate files accessible by other tasks and available for download in the UI's Output tab.", code = { "commands:", "- echo \"1\" >> {{ outputDir }}/first.txt", @@ -65,14 +59,14 @@ } ), @Example( - title = "Shell with an input file from Kestra's local storage created by a previous task.", + title = "Execute a Shell command using an input file generated in a previous task.", code = { "commands:", " - cat {{ outputs.previousTaskId.uri }}" } ), @Example( - title = "Run a command on a docker image", + title = "Run a PHP Docker container and execute a command", code = { "runner: DOCKER", "docker:", @@ -82,14 +76,14 @@ } ), @Example( - title = "Set outputs from bash standard output", + title = "Create output variables from a standard output", code = { "commands:", " - echo '::{\"outputs\":{\"test\":\"value\",\"int\":2,\"bool\":true,\"float\":3.65}}::'", } ), @Example( - title = "Send a counter metric from bash standard output", + title = "Send a counter metric from a standard output", code = { "commands:", " - echo '::{\"metrics\":[{\"name\":\"count\",\"type\":\"counter\",\"value\":1,\"tags\":{\"tag1\":\"i\",\"tag2\":\"win\"}}]}::'", diff --git a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Script.java b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Script.java index 5d1bec11..7d84d399 100644 --- a/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Script.java +++ b/plugin-script-shell/src/main/java/io/kestra/plugin/scripts/shell/Script.java @@ -26,7 +26,7 @@ @Plugin( examples = { @Example( - title = "Run a shell inline script.", + title = "Create an inline Shell script and execute it", code = { "script: |", " echo \"The current execution is : {{ execution.id }}\"", diff --git a/plugin-script/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java b/plugin-script/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java index ebae692b..dd15e782 100644 --- a/plugin-script/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java +++ b/plugin-script/src/main/java/io/kestra/core/tasks/scripts/AbstractBash.java @@ -37,7 +37,7 @@ abstract public class AbstractBash extends Task { @Builder.Default @Schema( - title = "Runner to use" + title = "The script runner — by default, Kestra runs all scripts in `DOCKER`." ) @PluginProperty @NotNull @@ -45,14 +45,14 @@ abstract public class AbstractBash extends Task { protected RunnerType runner = RunnerType.PROCESS; @Schema( - title = "Docker options when using runner `DOCKER`" + title = "Docker options when using the `DOCKER` runner." ) @PluginProperty protected DockerOptions dockerOptions; @Builder.Default @Schema( - title = "Interpreter to used" + title = "Interpreter to use when launching the process." ) @PluginProperty @NotNull @@ -68,10 +68,9 @@ abstract public class AbstractBash extends Task { @Builder.Default @Schema( - title = "Exit if any non true return value", + title = "Exit if any non true value is returned", description = "This tells bash that it should exit the script if any statement returns a non-true return value. \n" + - "The benefit of using -e is that it prevents errors snowballing into serious issues when they could " + - "have been caught earlier." + "Setting this to `true` helps catch cases where a command fails and the script continues to run anyway." ) @PluginProperty @NotNull @@ -106,8 +105,8 @@ abstract public class AbstractBash extends Task { protected List outputFiles; @Schema( - title = "Output dirs list that will be uploaded to internal storage", - description = "List of key that will generate temporary directories.\n" + + title = "List of output directories that will be uploaded to internal storage", + description = "List of keys that will generate temporary directories.\n" + "On the command, just can use with special variable named `outputDirs.key`.\n" + "If you add a files with `[\"myDir\"]`, you can use the special vars `echo 1 >> {[ outputDirs.myDir }}/file1.txt` " + "and `echo 2 >> {[ outputDirs.myDir }}/file2.txt` and both files will be uploaded to internal storage." + @@ -117,9 +116,10 @@ abstract public class AbstractBash extends Task { protected List outputDirs; @Schema( - title = "Input files are extra files that will be available in the script working directory.", - description = "You can define the files as map or a JSON string." + - "Each file can be defined inlined or can reference a file from Kestra's internal storage." + title = "Input files are extra files that will be available in the script's working directory.", + description = "Define the files **as a map** of a file name being the key, and the value being the file's content.\n" + + "Alternatively, configure the files **as a JSON string** with the same key/value structure as the map.\n" + + "In both cases, you can either specify the file's content inline, or reference a file from Kestra's internal storage by its URI, e.g. a file from an input, output of a previous task, or a [namespace file](https://kestra.io/docs/developer-guide/namespace-files)." ) @PluginProperty( additionalProperties = String.class, @@ -128,7 +128,7 @@ abstract public class AbstractBash extends Task { protected Object inputFiles; @Schema( - title = "Additional environments variable to add for current process." + title = "One or more additional environment variable(s) to add to the task run." ) @PluginProperty( additionalProperties = String.class, @@ -138,7 +138,7 @@ abstract public class AbstractBash extends Task { @Builder.Default @Schema( - title = "Use `WARNING` state if any stdErr is sent" + title = "Whether to set the execution state in `WARNING` if any `stdErr` is sent." ) @PluginProperty @NotNull diff --git a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java index 180537f3..50b7f048 100644 --- a/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java +++ b/plugin-script/src/main/java/io/kestra/plugin/scripts/exec/AbstractExecScript.java @@ -25,7 +25,7 @@ public abstract class AbstractExecScript extends Task implements RunnableTask, NamespaceFilesInterface, InputFilesInterface, OutputFilesInterface { @Builder.Default @Schema( - title = "Runner to use" + title = "Which script runner to use — by default, Kestra runs all scripts in `DOCKER`." ) @PluginProperty @NotNull @@ -33,7 +33,7 @@ public abstract class AbstractExecScript extends Task implements RunnableTask beforeCommands;