diff --git a/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/Eval.java b/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/Eval.java index 24ae496..4a1202f 100644 --- a/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/Eval.java +++ b/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/Eval.java @@ -21,50 +21,56 @@ full = true, title = "Make an API call and pass request body to a Groovy script.", code = """ - id: api-request-to-groovy - namespace: company.team - - tasks: - - id: request - type: io.kestra.plugin.core.http.Request - uri: "https://dummyjson.com/products/1" - - - id: groovy - type: io.kestra.plugin.scripts.groovy.Eval - script: | - logger.info('{{ outputs.request.body }}') - - - id: download - type: io.kestra.plugin.core.http.Download - uri: "https://dummyjson.com/products/1" - - - id: runContextGroovy - type: io.kestra.plugin.scripts.groovy.Eval - script: | - // logger.info('Vars: {}', runContext.getVariables()) - URI uri = new URI(runContext.variables.outputs.download.uri) - InputStream istream = runContext.storage().getFile(uri) - logger.info('Content: {}', istream.text) - """ + id: api_request_to_groovy + namespace: company.team + + tasks: + - id: request + type: io.kestra.plugin.core.http.Request + uri: "https://dummyjson.com/products/1" + + - id: groovy + type: io.kestra.plugin.scripts.groovy.Eval + script: | + logger.info('{{ outputs.request.body }}') + + - id: download + type: io.kestra.plugin.core.http.Download + uri: "https://dummyjson.com/products/1" + + - id: run_context_groovy + type: io.kestra.plugin.scripts.groovy.Eval + script: | + // logger.info('Vars: {}', runContext.getVariables()) + URI uri = new URI(runContext.variables.outputs.download.uri) + InputStream istream = runContext.storage().getFile(uri) + logger.info('Content: {}', istream.text) + """ ), @Example( - code = { - "outputs:", - " - out", - " - map", - "script: |", - " import io.kestra.core.models.executions.metrics.Counter", - " ", - " logger.info('executionId: {}', runContext.render('{{ execution.id }}'))", - " runContext.metric(Counter.of('total', 666, 'name', 'bla'))", - " ", - " map = Map.of('test', 'here')", - " File tempFile = runContext.workingDir().createTempFile().toFile()", - " var output = new FileOutputStream(tempFile)", - " output.write('555\\n666\\n'.getBytes())", - " ", - " out = runContext.storage().putFile(tempFile)" - } + code = """ + id: groovy_eval + namespace: company.team + + tasks: + - id: eval + type: io.kestra.plugin.scripts.groovy.Eval + outputs: + - out + - map + script: | + import io.kestra.core.models.executions.metrics.Counter + + logger.info('executionId: {}', runContext.render('{{ execution.id }}')) + runContext.metric(Counter.of('total', 666, 'name', 'bla')) + + map = Map.of('test', 'here') + File tempFile = runContext.workingDir().createTempFile().toFile() + var output = new FileOutputStream(tempFile) + output.write('555\\n666\\n'.getBytes()) + + out = runContext.storage().putFile(tempFile + """ ) } ) diff --git a/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/FileTransform.java b/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/FileTransform.java index ebba610..693044b 100644 --- a/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/FileTransform.java +++ b/plugin-script-groovy/src/main/java/io/kestra/plugin/scripts/groovy/FileTransform.java @@ -18,66 +18,100 @@ examples = { @Example( title = "Convert row by row of a file from Kestra's internal storage.", - code = { - "from: \"{{ outputs['avro-to-gcs'] }}\"", - "script: |", - " logger.info('row: {}', row)", - "", - " if (row.get('name') == 'richard') {", - " row = null", - " } else {", - " row.put('email', row.get('name') + '@kestra.io')", - " }" - } + full = true, + code = """ + id: groovy_file_transform + namespace: company.team + + inputs: + - id: file + type: FILE + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.groovy.FileTransform + from: "{{ inputs.file }}" + script: | + logger.info('row: {}', row) + + if (row.get('name') == 'richard') { + row = null + } else { + row.put('email', row.get('name') + '@kestra.io') + } + """ ), @Example( title = "Create multiple rows from one row.", - code = { - "from: \"{{ outputs['avro-to-gcs'] }}\"", - "script: |", - " logger.info('row: {}', row)", - " rows = [[\"action\", \"insert\"], row]" - } + full = true, + code = """ + id: groovy_file_transform + namespace: company.team + + inputs: + - id: file + type: FILE + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.groovy.FileTransform + from: "{{ inputs.file }}" + script: | + logger.info('row: {}', row) + rows = [["action", "insert"], row] + """ ), @Example( title = "Transform a JSON string to a file.", - code = { - "from: \"[{\\\"name\\\":\\\"jane\\\"}, {\\\"name\\\":\\\"richard\\\"}]\"", - "script: |", - " logger.info('row: {}', row)", - "", - " if (row.get('name') == 'richard') {", - " row = null", - " } else {", - " row.put('email', row.get('name') + '@kestra.io')", - " }" - } + full = true, + code = """ + id: groovy_file_transform + namespace: company.team + + inputs: + - id: json + type: JSON + defaults: [{"name":"jane"}, {"name":"richard"}] + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.groovy.FileTransform + from: "{{ inputs.json }}" + script: | + logger.info('row: {}', row) + + if (row.get('name') == 'richard') { + row = null + } else { + row.put('email', row.get('name') + '@kestra.io') + } + """ ), @Example( title = "JSON transformations using jackson library", full = true, code = """ -id: json_transform_using_jackson -namespace: company.team - -tasks: - - id: file_transform - type: io.kestra.plugin.scripts.groovy.FileTransform - from: "[{\"name\":\"John Doe\", \"age\":99, \"embedded\":{\"foo\":\"bar\"}}]" - script: | - import com.fasterxml.jackson.* - - def mapper = new databind.ObjectMapper(); - def jsonStr = mapper.writeValueAsString(row); - logger.info('input in json str: {}', jsonStr) - - def typeRef = new core.type.TypeReference>() {}; - - data = mapper.readValue(jsonStr, typeRef); - - logger.info('json object: {}', data); - logger.info('embedded field: {}', data.embedded.foo) -""" + id: json_transform_using_jackson + namespace: company.team + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.groovy.FileTransform + from: "[{\"name\":\"John Doe\", \"age\":99, \"embedded\":{\"foo\":\"bar\"}}]" + script: | + import com.fasterxml.jackson.* + + def mapper = new databind.ObjectMapper(); + def jsonStr = mapper.writeValueAsString(row); + logger.info('input in json str: {}', jsonStr) + + def typeRef = new core.type.TypeReference>() {}; + + data = mapper.readValue(jsonStr, typeRef); + + logger.info('json object: {}', data); + logger.info('embedded field: {}', data.embedded.foo) + """ ) } ) diff --git a/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Commands.java b/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Commands.java index 903f153..bbd0c40 100644 --- a/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Commands.java +++ b/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Commands.java @@ -28,17 +28,16 @@ @Example( title = "Execute JBang command to execute a JAR file.", full = true, - code = { + code = """ + id: jbang_commands + namespace: company.team + + tasks: + - id: commands + type: io.kestra.plugin.scripts.jbang.Commands + commands: + - jbang --quiet --main picocli.codegen.aot.graalvm.ReflectionConfigGenerator info.picocli:picocli-codegen:4.6.3 """ - id: jbang - namespace: company.team - - tasks: - - id: hello-jar - type: io.kestra.plugin.scripts.jbang.Commands - commands: - - jbang --quiet --main picocli.codegen.aot.graalvm.ReflectionConfigGenerator info.picocli:picocli-codegen:4.6.3""" - } ) } ) diff --git a/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Script.java b/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Script.java index 728ad95..4116b9e 100644 --- a/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Script.java +++ b/plugin-script-jbang/src/main/java/io/kestra/plugin/scripts/jbang/Script.java @@ -36,11 +36,11 @@ title = "Execute a script written in Java", full = true, code = """ - id: jbang + id: jbang_script namespace: company.team tasks: - - id: hello-java + - id: script type: io.kestra.plugin.scripts.jbang.Script script: | class helloworld { @@ -51,49 +51,52 @@ public static void main(String[] args) { System.out.println("Hello " + args[0]); } } - }""" + } + """ ), @Example( title = "Execute a script written in Java with dependencies", full = true, code = """ - id: jbang + id: jbang_script namespace: company.team tasks: - - id: hello-dependency - type: io.kestra.plugin.scripts.jbang.Script - script: | - //DEPS ch.qos.reload4j:reload4j:1.2.19 - - import org.apache.log4j.Logger; - import org.apache.log4j.BasicConfigurator; - - class classpath_example { - - static final Logger logger = Logger.getLogger(classpath_example.class); - - public static void main(String[] args) { - BasicConfigurator.configure();\s - logger.info("Hello World"); + - id: script_with_dependency + type: io.kestra.plugin.scripts.jbang.Script + script: | + //DEPS ch.qos.reload4j:reload4j:1.2.19 + + import org.apache.log4j.Logger; + import org.apache.log4j.BasicConfigurator; + + class classpath_example { + + static final Logger logger = Logger.getLogger(classpath_example.class); + + public static void main(String[] args) { + BasicConfigurator.configure();\s + logger.info("Hello World"); + } } - }""" + """ ), @Example( - title = "Execute a script written in Kotlin", + title = "Execute a script written in Kotlin.", full = true, code = """ - id: jbang + id: jbang_script namespace: company.team tasks: - - id: hello-kotlin + - id: script_kotlin type: io.kestra.plugin.scripts.jbang.Script extension: .kt script: | public fun main() { println("Hello World"); - }""" + } + """ ) } ) diff --git a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java index b311427..0f5e103 100644 --- a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java +++ b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Commands.java @@ -28,10 +28,11 @@ full = true, title = "Create a Julia script, install required packages and execute it. Note that instead of defining the script inline, you could create the Julia script in the embedded VS Code editor and point to its location by path. If you do so, make sure to enable namespace files by setting the `enabled` flag of the `namespaceFiles` property to `true`.", code = """ - id: script + id: julia_commands namespace: company.team + tasks: - - id: bash + - id: commands type: io.kestra.plugin.scripts.julia.Commands warningOnStdErr: false inputFiles: diff --git a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java index 87bb0c3..9bdd8be 100644 --- a/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java +++ b/plugin-script-julia/src/main/java/io/kestra/plugin/scripts/julia/Script.java @@ -34,10 +34,11 @@ full = true, title = "Create a Julia script, install required packages and execute it. Note that instead of defining the script inline, you could create the Julia script in the embedded VS Code editor and read its content using the `{{ read('your_script.jl') }}` function.", code = """ - id: script + id: julia_script namespace: company.team + tasks: - - id: bash + - id: script type: io.kestra.plugin.scripts.julia.Script warningOnStdErr: false script: | diff --git a/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/Eval.java b/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/Eval.java index 03e12ef..ea3b5b0 100644 --- a/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/Eval.java +++ b/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/Eval.java @@ -18,24 +18,31 @@ @Plugin( examples = { @Example( - code = { - "outputs:", - " - out", - " - map", - "script: |", - " from io.kestra.core.models.executions.metrics import Counter", - " import tempfile", - " from java.io import File", - " ", - " logger.info('executionId: {}', runContext.render('{{ execution.id }}'))", - " runContext.metric(Counter.of('total', 666, 'name', 'bla'))", - " ", - " map = {'test': 'here'}", - " tempFile = tempfile.NamedTemporaryFile()", - " tempFile.write('555\\n666\\n')", - " ", - " out = runContext.storage().putFile(File(tempFile.name))" - } + full = true, + code = """ + id: jython_eval + namespace: company.team + + tasks: + - id: eval + type: io.kestra.plugin.scripts.jython.Eval + outputs: + - out + - map + script: | + from io.kestra.core.models.executions.metrics import Counter + import tempfile + from java.io import File + + logger.info('executionId: {}', runContext.render('{{ execution.id }}')) + runContext.metric(Counter.of('total', 666, 'name', 'bla')) + + map = {'test': 'here'} + tempFile = tempfile.NamedTemporaryFile() + tempFile.write('555\\n666\\n') + + out = runContext.storage().putFile(File(tempFile.name) + """ ) } ) diff --git a/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/FileTransform.java b/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/FileTransform.java index 4443f98..1665bd9 100644 --- a/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/FileTransform.java +++ b/plugin-script-jython/src/main/java/io/kestra/plugin/scripts/jython/FileTransform.java @@ -20,61 +20,84 @@ full = true, title = "Extract data from an API, add a column, and store it as a downloadable CSV file.", code = """ -id: etl-api-to-csv -namespace: company.team - -tasks: - - id: download - type: io.kestra.plugin.fs.http.Download - uri: https://gorest.co.in/public/v2/users - - - id: ionToJSON - type: io.kestra.plugin.serdes.json.JsonToIon - from: "{{outputs.download.uri}}" - newLine: false - - - id: writeJSON - type: io.kestra.plugin.serdes.json.IonToJson - from: "{{outputs.ionToJSON.uri}}" - - - id: addColumn - type: io.kestra.plugin.scripts.jython.FileTransform - from: "{{outputs.writeJSON.uri}}" - script: | - from datetime import datetime - logger.info('row: {}', row) - row['inserted_at'] = datetime.utcnow() - - - id: csv - type: io.kestra.plugin.serdes.csv.IonToCsv - from: "{{outputs.addColumn.uri}}" -""" + id: etl_api_to_csv + namespace: company.team + + tasks: + - id: download + type: io.kestra.plugin.fs.http.Download + uri: https://gorest.co.in/public/v2/users + + - id: ion_to_json + type: io.kestra.plugin.serdes.json.JsonToIon + from: "{{ outputs.download.uri }}" + newLine: false + + - id: write_json + type: io.kestra.plugin.serdes.json.IonToJson + from: "{{ outputs.ion_to_json.uri }}" + + - id: add_column + type: io.kestra.plugin.scripts.jython.FileTransform + from: "{{ outputs.write_json.uri }}" + script: | + from datetime import datetime + logger.info('row: {}', row) + row['inserted_at'] = datetime.utcnow() + + - id: csv + type: io.kestra.plugin.serdes.csv.IonToCsv + from: "{{ outputs.add_column.uri }}" + """ ), @Example( title = "Transform with file from internal storage.", - code = { - "from: \"{{ outputs['avro-to-gcs'] }}\"", - "script: |", - " logger.info('row: {}', row)", - "", - " if row['name'] == 'richard': ", - " row = None", - " else: ", - " row['email'] = row['name'] + '@kestra.io'\n" - } + full = true, + code = """ + id: jython_file_transform + namespace: company.team + + inputs: + - id: file + type: FILE + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.jython.FileTransform + from: "{{ inputs.file }}" + script: | + logger.info('row: {}', row) + + if row['name'] == 'richard': + row = None + else: + row['email'] = row['name'] + '@kestra.io' + """ ), @Example( title = "Transform with file from JSON string.", - code = { - "from: \"[{\\\"name\\\":\\\"jane\\\"}, {\\\"name\\\":\\\"richard\\\"}]\"", - "script: |", - " logger.info('row: {}', row)", - "", - " if row['name'] == 'richard': ", - " row = None", - " else: ", - " row['email'] = row['name'] + '@kestra.io'\n" - } + full = true, + code = """ + id: jython_file_transform + namespace: company.team + + inputs: + - id: json + type: JSON + defaults: {"name": "john"} + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.jython.FileTransform + from: "{{ inputs.json }}" + script: | + logger.info('row: {}', row) + + if row['name'] == 'richard': + row = None + else: + row['email'] = row['name'] + '@kestra.io' + """ ) } ) diff --git a/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/Eval.java b/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/Eval.java index 1c15a7b..5e3989c 100644 --- a/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/Eval.java +++ b/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/Eval.java @@ -18,25 +18,32 @@ @Plugin( examples = { @Example( - code = { - "outputs:", - " - out", - " - map", - "script: |", - " var Counter = Java.type('io.kestra.core.models.executions.metrics.Counter');", - " var File = Java.type('java.io.File');", - " var FileOutputStream = Java.type('java.io.FileOutputStream');", - " ", - " logger.info('executionId: {}', runContext.render('{{ execution.id }}'));", - " runContext.metric(Counter.of('total', 666, 'name', 'bla'));", - " ", - " map = {'test': 'here'}", - " var tempFile = runContext.workingDir().createTempFile().toFile()", - " var output = new FileOutputStream(tempFile)", - " output.write('555\\n666\\n'.getBytes())", - " ", - " out = runContext.storage().putFile(tempFile)" - } + full = true, + code = """ + id: nashorn_eval + namespace: company.team + + tasks: + - id: eval + type: io.kestra.plugin.scripts.nashorn.Eval + outputs: + - out + - map + script: | + var Counter = Java.type('io.kestra.core.models.executions.metrics.Counter'); + var File = Java.type('java.io.File'); + var FileOutputStream = Java.type('java.io.FileOutputStream'); + + logger.info('executionId: {}', runContext.render('{{ execution.id }}')); + runContext.metric(Counter.of('total', 666, 'name', 'bla')); + + map = {'test': 'here'} + var tempFile = runContext.workingDir().createTempFile().toFile() + var output = new FileOutputStream(tempFile) + output.write('555\\n666\\n'.getBytes()) + + out = runContext.storage().putFile(tempFile)" + """ ) } ) diff --git a/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/FileTransform.java b/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/FileTransform.java index e9e0be1..a5b884a 100644 --- a/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/FileTransform.java +++ b/plugin-script-nashorn/src/main/java/io/kestra/plugin/scripts/nashorn/FileTransform.java @@ -19,31 +19,45 @@ examples = { @Example( title = "Transform with file from internal storage", - code = { - "from: \"{{ outputs['avro-to-gcs'] }}\"", - "script: |", - " logger.info('row: {}', row)", - "", - " if (row['name'] === 'richard') {", - " row = null", - " } else {", - " row['email'] = row['name'] + '@kestra.io'", - " }" - } + full = true, + code = """ + id: nashorn_file_transform + namespace: company.team + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.nashorn.FileTransform + from: "{{ outputs['avro-to-gcs'] }}" + script: | + logger.info('row: {}', row) + + if (row['name'] === 'richard') { + row = null + } else { + row['email'] = row['name'] + '@kestra.io' + } + """ ), @Example( title = "Transform JSON string input with a Nashorn script.", - code = { - "from: \"[{\\\"name\":\\\"jane\\\"}, {\\\"name\\\":\\\"richard\\\"}]\"", - "script: |", - " logger.info('row: {}', row)", - "", - " if (row['name'] === 'richard') {", - " row = null", - " } else {", - " row['email'] = row['name'] + '@kestra.io'", - " }" - } + full = true, + code = """ + id: nashorn_file_transform + namespace: company.team + + tasks: + - id: file_transform + type: io.kestra.plugin.scripts.nashorn.FileTransform + from: "[{\"name":\"jane\"}, {\"name\":\"richard\"}]" + script: | + logger.info('row: {}', row) + + if (row['name'] === 'richard') { + row = null + } else { + row['email'] = row['name'] + '@kestra.io' + } + """ ) } ) diff --git a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java index 500f543..d2bd850 100644 --- a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java +++ b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Commands.java @@ -28,10 +28,11 @@ full = true, title = "Install required npm packages, create a Node.js script and execute it.", code = """ - id: node + id: nodejs_commands namespace: company.team + tasks: - - id: node_script + - id: commands type: io.kestra.plugin.scripts.node.Commands inputFiles: main.js: | diff --git a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java index 651e409..550a633 100644 --- a/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java +++ b/plugin-script-node/src/main/java/io/kestra/plugin/scripts/node/Script.java @@ -32,14 +32,21 @@ @Plugin(examples = { @Example( title = "Install package, create a Node.js script and execute it.", - code = { - "beforeCommands:", - " - npm install colors", - "script: |", - " const colors = require(\"colors\");", - " console.log(colors.red(\"Hello\"));", - "warningOnStdErr: false", - } + full = true, + code = """ + id: nodejs_script + namespace: company.team + + tasks: + - id: script + type: io.kestra.plugin.scripts.node.Script + beforeCommands: + - npm install colors + script: | + const colors = require("colors"); + console.log(colors.red("Hello")); + warningOnStdErr: false" + """ ), @Example( full = true, @@ -49,8 +56,9 @@ Alternatively, instead of the `{{ outputDir }}` variable, you could use the `outputFiles` property to output files from your script. You can access those files in downstream tasks using the same syntax `{{ outputs.yourTaskId.outputFiles['yourFileName.fileExtension'] }}`, and you can download the files from the UI's Output tab. """, code = """ - id: nodeJS + id: nodejs_script namespace: company.team + tasks: - id: node type: io.kestra.plugin.scripts.node.Script