diff --git a/src/main/java/io/kestra/plugin/soda/AbstractSoda.java b/src/main/java/io/kestra/plugin/soda/AbstractSoda.java index 363ac9e..a68c41e 100644 --- a/src/main/java/io/kestra/plugin/soda/AbstractSoda.java +++ b/src/main/java/io/kestra/plugin/soda/AbstractSoda.java @@ -48,9 +48,8 @@ public abstract class AbstractSoda extends Task { title = "Runner to use", description = "Deprecated, use 'taskRunner' instead" ) - @PluginProperty @Deprecated - protected RunnerType runner; + protected Property runner; @Schema( title = "Deprecated, use 'taskRunner' instead" @@ -69,9 +68,8 @@ public abstract class AbstractSoda extends Task { private TaskRunner taskRunner = Docker.instance(); @Schema(title = "The task runner container image, only used if the task runner is container-based.") - @PluginProperty(dynamic = true) @Builder.Default - private String containerImage = DEFAULT_IMAGE; + private Property containerImage = Property.of(DEFAULT_IMAGE); @Schema(title = "Deprecated, use the `docker` property instead", deprecated = true) @PluginProperty @@ -99,7 +97,6 @@ public void setDockerOptions(DockerOptions dockerOptions) { title = "List of python dependencies to add to the python execution process", description = "Python dependencies list to setup in the virtualenv, in the same format than requirements.txt. It must at least provides dbt." ) - @NotNull protected Property> requirements; @Schema( @@ -110,15 +107,16 @@ public void setDockerOptions(DockerOptions dockerOptions) { @Schema( title = "The configuration file" ) - @PluginProperty(dynamic = true) @NotNull - @NotEmpty - Map configuration; + Property<@NotEmpty Map> configuration; protected Map finalInputFiles(RunContext runContext, Path workingDirectory) throws IOException, IllegalVariableEvaluationException { Map map = this.inputFiles != null ? new HashMap<>(PluginUtilsService.transformInputFiles(runContext, this.inputFiles)) : new HashMap<>(); - map.put("configuration.yml", MAPPER.writeValueAsString(runContext.render(configuration))); + var renderedConfig = runContext.render(configuration).asMap(String.class, Object.class); + if (!renderedConfig.isEmpty()) { + map.put("configuration.yml", MAPPER.writeValueAsString(renderedConfig)); + } return map; } @@ -127,9 +125,9 @@ public CommandsWrapper start(RunContext runContext) throws Exception { var env = runContext.render(this.getEnv()).asMap(String.class, String.class); CommandsWrapper commandsWrapper = new CommandsWrapper(runContext) .withEnv(env.isEmpty() ? new HashMap<>() : env) - .withRunnerType(this.getRunner()) + .withRunnerType(runContext.render(this.getRunner()).as(RunnerType.class).orElse(null)) .withTaskRunner(this.taskRunner) - .withContainerImage(this.getContainerImage()) + .withContainerImage(runContext.render(this.getContainerImage()).as(String.class).orElse(null)) .withOutputFiles(List.of("result.json")) .withDockerOptions(injectDefaults(this.getDocker())); Path workingDirectory = commandsWrapper.getWorkingDirectory(); diff --git a/src/main/java/io/kestra/plugin/soda/Scan.java b/src/main/java/io/kestra/plugin/soda/Scan.java index a3450c8..1474777 100644 --- a/src/main/java/io/kestra/plugin/soda/Scan.java +++ b/src/main/java/io/kestra/plugin/soda/Scan.java @@ -143,7 +143,7 @@ public Scan.Output run(RunContext runContext) throws Exception { .result(scanResult) .stdOutLineCount(output.getStdOutLineCount()) .stdErrLineCount(output.getStdOutLineCount()) - .configuration(runContext.render(configuration)) + .configuration(runContext.render(configuration).asMap(String.class, Object.class)) .exitCode((Integer) output.getVars().get("exitCode")) .build(); } diff --git a/src/test/java/io/kestra/plugin/soda/ScanTest.java b/src/test/java/io/kestra/plugin/soda/ScanTest.java index 4ea54c2..7e7a37a 100644 --- a/src/test/java/io/kestra/plugin/soda/ScanTest.java +++ b/src/test/java/io/kestra/plugin/soda/ScanTest.java @@ -34,7 +34,7 @@ void run() throws Exception { .id("unit-test") .type(Scan.class.getName()) .taskRunner(Docker.builder().type(Docker.class.getName()).build()) - .configuration(JacksonMapper.ofYaml().readValue( + .configuration(Property.of(JacksonMapper.ofYaml().readValue( "data_source kestra:\n" + " type: bigquery\n" + " connection:\n" + @@ -43,7 +43,7 @@ void run() throws Exception { " account_info_json: |\n" + " " + StringUtils.replace(UtilsTest.serviceAccount(), "\n", "\n "), TYPE_REFERENCE - )) + ))) .checks(JacksonMapper.ofYaml().readValue("checks for orderDetail:\n" + " - row_count > 0\n" + " - max(unitPrice):\n" + @@ -70,7 +70,7 @@ void failed() throws Exception { .id("unit-test") .type(Scan.class.getName()) .taskRunner(Docker.builder().type(Docker.class.getName()).build()) - .configuration(JacksonMapper.ofYaml().readValue( + .configuration(Property.of(JacksonMapper.ofYaml().readValue( "data_source kestra:\n" + " type: bigquery\n" + " connection:\n" + @@ -79,7 +79,7 @@ void failed() throws Exception { " account_info_json: |\n" + " " + StringUtils.replace(UtilsTest.serviceAccount(), "\n", "\n "), TYPE_REFERENCE - )) + ))) .checks(JacksonMapper.ofYaml().readValue("checks for territory:\n" + " - row_count > 0\n" + " - failed rows:\n" + @@ -104,7 +104,7 @@ void error() throws Exception { .id("unit-test") .type(Scan.class.getName()) .taskRunner(Docker.builder().type(Docker.class.getName()).build()) - .configuration(JacksonMapper.ofYaml().readValue( + .configuration(Property.of(JacksonMapper.ofYaml().readValue( "data_source kestra:\n" + " type: bigquery\n" + " connection:\n" + @@ -113,7 +113,7 @@ void error() throws Exception { " account_info_json: |\n" + " " + StringUtils.replace(UtilsTest.serviceAccount(), "\n", "\n "), TYPE_REFERENCE - )) + ))) .checks(JacksonMapper.ofYaml().readValue("checks for orderDetail:\n" + "checks for invalid_table:\n" + " - row_count > 0",