diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ada65fd..ef712d072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,21 @@ +# Viash 0.8.0-RC3 (2023-10-07): More bugfixes, more Nextflow functionality + +This release contains more bugfixes related to the Nextflow code generation functionality. +It also adds a new `runIf` argument to the `NextflowPlatform` which allows for conditional execution of modules. + +## NEW FUNCTIONALITY + +* `NextflowPlatform`: Added new `.run()` argument `runIf` - a function that determines whether the module should be run or not (PR #553). + If the `runIf` closure evaluates to `true`, then the module will be run. Otherwise it will be passed through without running. + +## MINOR CHANGES + +* `NextflowPlatform`: Rename internal VDSL3 variables to be more consistent with regular Viash component variables and avoid naming clashes (PR #553). + +## BUG FIXES + +* `export cli_autocomplete`: Fix output script format and hide `--loglevel` and `--colorize` (PR #544). Masked arguments are usable but might not be very useful to always display in help messages. + # Viash 0.8.0-RC2 (2023-10-04): Some bugfixes Some bugfixes related to the new dependencies and Nextflow code generation functionality. diff --git a/build.sbt b/build.sbt index 14de083d6..632444167 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "viash" -version := "0.8.0-RC2" +version := "0.8.0-RC3" scalaVersion := "2.13.10" diff --git a/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf b/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf index eb2961fb0..b774bfffb 100644 --- a/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf +++ b/src/main/resources/io/viash/platforms/nextflow/VDSL3Helper.nf @@ -11,7 +11,7 @@ * Internally, this workflow will convert the input channel * to a format which the Nextflow module will be able to handle. */ -def vdsl3WorkflowFactory(Map args) { +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { def key = args["key"] def processObj = null @@ -20,7 +20,7 @@ def vdsl3WorkflowFactory(Map args) { main: if (processObj == null) { - processObj = _vdsl3ProcessFactory(args) + processObj = _vdsl3ProcessFactory(args, meta, rawScript) } output_ = input_ @@ -34,7 +34,7 @@ def vdsl3WorkflowFactory(Map args) { } // process input files separately - def inputPaths = thisConfig.functionality.allArguments + def inputPaths = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "input" } .collect { par -> def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] @@ -62,7 +62,7 @@ def vdsl3WorkflowFactory(Map args) { } // remove input files - def argsExclInputFiles = thisConfig.functionality.allArguments + def argsExclInputFiles = meta.config.functionality.allArguments .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } .collectEntries { par -> def parName = par.plainName @@ -76,11 +76,11 @@ def vdsl3WorkflowFactory(Map args) { [parName, val] } - [ id ] + inputPaths + [ argsExclInputFiles, resourcesDir ] + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] } | processObj | map { output -> - def outputFiles = thisConfig.functionality.allArguments + def outputFiles = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "output" } .indexed() .collectEntries{ index, par -> @@ -114,13 +114,13 @@ def vdsl3WorkflowFactory(Map args) { return processWf } -// depends on: thisConfig, thisScript, session? -def _vdsl3ProcessFactory(Map workflowArgs) { +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { // autodetect process key def wfKey = workflowArgs["key"] def procKeyPrefix = "${wfKey}_process" - def meta = nextflow.script.ScriptMeta.current() - def existing = meta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} def newNumber = (numbers + [-1]).max() + 1 @@ -174,12 +174,12 @@ def _vdsl3ProcessFactory(Map workflowArgs) { } }.join() - def inputPaths = thisConfig.functionality.allArguments + def inputPaths = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "input" } .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } .join() - def outputPaths = thisConfig.functionality.allArguments + def outputPaths = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "output" } .collect { par -> // insert dummy into every output (see nextflow-io/nextflow#2678) @@ -199,7 +199,7 @@ def _vdsl3ProcessFactory(Map workflowArgs) { } // create dirs for output files (based on BashWrapper.createParentFiles) - def createParentStr = thisConfig.functionality.allArguments + def createParentStr = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "output" && it.create_parent } .collect { par -> "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"] : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" @@ -207,7 +207,7 @@ def _vdsl3ProcessFactory(Map workflowArgs) { .join("\n") // construct inputFileExports - def inputFileExports = thisConfig.functionality.allArguments + def inputFileExports = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } .collect { par -> viash_par_contents = "(viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName})" @@ -229,7 +229,7 @@ def _vdsl3ProcessFactory(Map workflowArgs) { ).toAbsolutePath() // construct stub - def stub = thisConfig.functionality.allArguments + def stub = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "output" } .collect { par -> "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" @@ -237,7 +237,7 @@ def _vdsl3ProcessFactory(Map workflowArgs) { .join("\n") // escape script - def escapedScript = thisScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') // publishdir assert def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? @@ -269,7 +269,7 @@ def _vdsl3ProcessFactory(Map workflowArgs) { |# export VIASH_META_RESOURCES_DIR="\${resourcesDir.toRealPath().toAbsolutePath()}" |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" - |export VIASH_META_FUNCTIONALITY_NAME="${thisConfig.functionality.name}" + |export VIASH_META_FUNCTIONALITY_NAME="${meta.config.functionality.name}" |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_FUNCTIONALITY_NAME" |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } @@ -312,16 +312,17 @@ def _vdsl3ProcessFactory(Map workflowArgs) { def ownerParams = new nextflow.script.ScriptBinding.ParamsMap() def binding = new nextflow.script.ScriptBinding().setParams(ownerParams) def module = new nextflow.script.IncludeDef.Module(name: procKey) + def session = nextflow.Nextflow.getSession() def scriptParser = new nextflow.script.ScriptParser(session) .setModule(true) .setBinding(binding) - scriptParser.scriptPath = meta.getScriptPath() + scriptParser.scriptPath = scriptMeta.getScriptPath() def moduleScript = scriptParser.runScript(procStr) .getScript() // register module in meta - meta.addModule(moduleScript, module.name, module.alias) + scriptMeta.addModule(moduleScript, module.name, module.alias) // retrieve and return process from meta - return meta.getProcess(procKey) + return scriptMeta.getProcess(procKey) } diff --git a/src/main/resources/io/viash/platforms/nextflow/arguments/processInputsOutputs.nf b/src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf similarity index 54% rename from src/main/resources/io/viash/platforms/nextflow/arguments/processInputsOutputs.nf rename to src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf index 5987bbc43..e407ac703 100644 --- a/src/main/resources/io/viash/platforms/nextflow/arguments/processInputsOutputs.nf +++ b/src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf @@ -1,4 +1,5 @@ -def typeCheck(String stage, Map par, Object value, String id, String key) { + +def _checkArgumentType(String stage, Map par, Object value, String id, String key) { // expectedClass will only be != null if value is not of the expected type def expectedClass = null @@ -8,7 +9,7 @@ def typeCheck(String stage, Map par, Object value, String id, String key) { if (value instanceof List) { try { value = value.collect { listVal -> - typeCheck(stage, par + [multiple: false], listVal, id, key) + _checkArgumentType(stage, par + [multiple: false], listVal, id, key) } } catch (Exception e) { expectedClass = "List[${par.type}]" @@ -61,45 +62,3 @@ def typeCheck(String stage, Map par, Object value, String id, String key) { return value } - -Map processInputs(Map inputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.functionality.allArguments.each { arg -> - if (arg.required) { - assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" - } - } - - inputs = inputs.collectEntries { name, value -> - def par = config.functionality.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" - - value = typeCheck("input", par, value, id, key) - - [ name, value ] - } - } - return inputs -} - -Map processOutputs(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.functionality.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - - outputs = outputs.collectEntries { name, value -> - def par = config.functionality.allArguments.find { it.plainName == name && it.direction == "output" } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" - - value = typeCheck("output", par, value, id, key) - - [ name, value ] - } - } - return outputs -} diff --git a/src/main/resources/io/viash/platforms/nextflow/arguments/_processInputValues.nf b/src/main/resources/io/viash/platforms/nextflow/arguments/_processInputValues.nf new file mode 100644 index 000000000..e7b1bd852 --- /dev/null +++ b/src/main/resources/io/viash/platforms/nextflow/arguments/_processInputValues.nf @@ -0,0 +1,20 @@ +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.functionality.allArguments.each { arg -> + if (arg.required) { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.functionality.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, id, key) + + [ name, value ] + } + } + return inputs +} diff --git a/src/main/resources/io/viash/platforms/nextflow/arguments/_processOutputValues.nf b/src/main/resources/io/viash/platforms/nextflow/arguments/_processOutputValues.nf new file mode 100644 index 000000000..d7caa0b91 --- /dev/null +++ b/src/main/resources/io/viash/platforms/nextflow/arguments/_processOutputValues.nf @@ -0,0 +1,20 @@ +Map _processOutputValues(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.functionality.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + + outputs = outputs.collectEntries { name, value -> + def par = config.functionality.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, id, key) + + [ name, value ] + } + } + return outputs +} diff --git a/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf b/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf index a97730bc6..a050a1587 100644 --- a/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf +++ b/src/main/resources/io/viash/platforms/nextflow/channel/runComponents.nf @@ -15,6 +15,7 @@ * @return: a workflow that runs the components **/ def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") assert args.components: "runComponents should be passed a list of components to run" def components_ = args.components diff --git a/src/main/resources/io/viash/platforms/nextflow/channel/runEach.nf b/src/main/resources/io/viash/platforms/nextflow/channel/runEach.nf new file mode 100644 index 000000000..d664d01fc --- /dev/null +++ b/src/main/resources/io/viash/platforms/nextflow/channel/runEach.nf @@ -0,0 +1,105 @@ +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_) + } + [new_id] + tup.drop(1) + } + : filter_ch + data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} diff --git a/src/main/resources/io/viash/platforms/nextflow/readwrite/toTaggedYamlBlob.nf b/src/main/resources/io/viash/platforms/nextflow/readwrite/toTaggedYamlBlob.nf index 03da31dd0..6d3e50952 100644 --- a/src/main/resources/io/viash/platforms/nextflow/readwrite/toTaggedYamlBlob.nf +++ b/src/main/resources/io/viash/platforms/nextflow/readwrite/toTaggedYamlBlob.nf @@ -1,16 +1,22 @@ // Custom representer to modify how certain objects are represented in YAML class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { public String getFileName(Object obj) { - if (obj instanceof Path) { - def file = (Path) obj; - return file.getFileName().toString(); - } else if (obj instanceof File) { - def file = (File) obj; - return file.getName(); - } else { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } } public org.yaml.snakeyaml.nodes.Node representData(Object data) { @@ -19,8 +25,9 @@ class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { return representScalar(tag, filename); } } - CustomRepresenter(org.yaml.snakeyaml.DumperOptions options) { + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { super(options) + this.relativizer = relativizer this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) this.representers.put(Path, new RepresentPath()) this.representers.put(File, new RepresentPath()) @@ -28,9 +35,12 @@ class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { } String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { def options = new org.yaml.snakeyaml.DumperOptions() options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - def representer = new CustomRepresenter(options) + def representer = new CustomRepresenter(options, relativizer) def yaml = new org.yaml.snakeyaml.Yaml(representer, options) return yaml.dump(data) } diff --git a/src/main/resources/io/viash/platforms/nextflow/readwrite/toYamlBlob.nf b/src/main/resources/io/viash/platforms/nextflow/readwrite/toYamlBlob.nf index 333e395f6..faeddb29c 100644 --- a/src/main/resources/io/viash/platforms/nextflow/readwrite/toYamlBlob.nf +++ b/src/main/resources/io/viash/platforms/nextflow/readwrite/toYamlBlob.nf @@ -3,6 +3,6 @@ String toYamlBlob(data) { options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) options.setPrettyFlow(true) def yaml = new org.yaml.snakeyaml.Yaml(options) - def cleanData = iterateMap(data, {it.toString()}) - return yaml.dump(data) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) } diff --git a/src/main/resources/io/viash/platforms/nextflow/states/joinStates.nf b/src/main/resources/io/viash/platforms/nextflow/states/joinStates.nf new file mode 100644 index 000000000..86ce6e395 --- /dev/null +++ b/src/main/resources/io/viash/platforms/nextflow/states/joinStates.nf @@ -0,0 +1,17 @@ +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} \ No newline at end of file diff --git a/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf b/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf index b32b8f769..c119cc7d7 100644 --- a/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf +++ b/src/main/resources/io/viash/platforms/nextflow/states/publishStates.nf @@ -40,6 +40,7 @@ def collectInputOutputPaths(obj, prefix) { def publishStates(Map args) { def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) assert key_ != null : "publishStates: key must be specified" @@ -56,17 +57,16 @@ def publishStates(Map args) { def inputFiles_ = inputOutputFiles_[0] def outputFiles_ = inputOutputFiles_[1] - // convert state to yaml blob - def yamlBlob_ = toTaggedYamlBlob([id: id_] + state_) - - // adds a leading dot to the id (after any folder names) - // example: foo -> .foo, foo/bar -> foo/.bar - def idWithDot_ = id_.replaceAll("^(.+/)?([^/]+)", "\$1.\$2") - def yamlFile = '$id.$key.state.yaml' - .replaceAll('\\$id', idWithDot_) + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) .replaceAll('\\$key', key_) - [id_, yamlBlob_, yamlFile, inputFiles_, outputFiles_] + // TODO: do the pathnames in state_ match up with the outputFiles_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFiles_] } | publishStatesProc emit: input_ch @@ -89,7 +89,10 @@ process publishStatesProc { .transpose() .collectMany{infile, outfile -> if (infile.toString() != outfile.toString()) { - ["cp -r '${infile.toString()}' '${outfile.toString()}'"] + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] } else { // no need to copy if infile is the same as outfile [] @@ -122,6 +125,14 @@ def publishStatesByConfig(Map args) { def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$key', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + // the processed state is a list of [key, value, srcPath, destPath] tuples, where // - key, value is part of the state to be saved to disk // - srcPath and destPath are lists of files to be copied from src to dest @@ -164,19 +175,26 @@ def publishStatesByConfig(Map args) { // the index of the file assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" def outputPerFile = value.withIndex().collect{ val, ix -> - def destPath = filename.replace("*", ix.toString()) - def destFile = java.nio.file.Paths.get(destPath) + def value_ = java.nio.file.Paths.get(filename.replace("*", ix.toString())) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } def srcPath = val instanceof File ? val.toPath() : val - [value: destFile, srcPath: srcPath, destPath: destPath] + [value: value_, srcPath: srcPath, destPath: destPath] } def transposedOutputs = ["value", "srcPath", "destPath"].collectEntries{ key -> [key, outputPerFile.collect{dic -> dic[key]}] } return [[key: plainName_] + transposedOutputs] } else { - def destFile = java.nio.file.Paths.get(filename) + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } def srcPath = value instanceof File ? value.toPath() : value - return [[key: plainName_, value: destFile, srcPath: [srcPath], destPath: [filename]]] + return [[key: plainName_, value: value_, srcPath: [srcPath], destPath: [filename]]] } } @@ -187,15 +205,7 @@ def publishStatesByConfig(Map args) { // convert state to yaml blob def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - // adds a leading dot to the id (after any folder names) - // example: foo -> .foo, foo/bar -> foo/.bar - // TODO: allow defining the state.yaml template - def idWithDot_ = id_.replaceAll("^(.+/)?([^/]+)", "\$1.\$2") - def yamlFile = '$id.$key.state.yaml' - .replaceAll('\\$id', idWithDot_) - .replaceAll('\\$key', key_) - - [id_, yamlBlob_, yamlFile, inputFiles_, outputFiles_] + [id_, yamlBlob_, yamlFilename, inputFiles_, outputFiles_] } | publishStatesProc emit: input_ch diff --git a/src/main/resources/io/viash/platforms/nextflow/workflowFactory/processWorkflowArgs.nf b/src/main/resources/io/viash/platforms/nextflow/workflowFactory/processWorkflowArgs.nf index d56d83544..9b1719665 100644 --- a/src/main/resources/io/viash/platforms/nextflow/workflowFactory/processWorkflowArgs.nf +++ b/src/main/resources/io/viash/platforms/nextflow/workflowFactory/processWorkflowArgs.nf @@ -1,33 +1,32 @@ -// depends on: thisConfig, thisDefaultWorkflowArgs -def processWorkflowArgs(Map args) { +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { // override defaults with args - def workflowArgs = thisDefaultWorkflowArgs + args + def workflowArgs = defaultWfArgs + args // check whether 'key' exists - assert workflowArgs.containsKey("key") : "Error in module '${thisConfig.functionality.name}': key is a required argument" + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.functionality.name}': key is a required argument" // if 'key' is a closure, apply it to the original key if (workflowArgs["key"] instanceof Closure) { - workflowArgs["key"] = workflowArgs["key"](thisConfig.functionality.name) + workflowArgs["key"] = workflowArgs["key"](meta.config.functionality.name) } def key = workflowArgs["key"] assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" // check for any unexpected keys - def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "fromState", "toState", "args", "renameKeys", "debug"] + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] def unexpectedKeys = workflowArgs.keySet() - expectedKeys assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" // check whether directives exists and apply defaults assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" - workflowArgs["directives"] = processDirectives(thisDefaultWorkflowArgs.directives + workflowArgs["directives"]) + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) // check whether directives exists and apply defaults assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" - workflowArgs["auto"] = processAuto(thisDefaultWorkflowArgs.auto + workflowArgs["auto"]) + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) // auto define publish, if so desired if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { @@ -75,24 +74,24 @@ def processWorkflowArgs(Map args) { workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) } - for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter"]) { + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" } } // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? - for (nam in ["renameKeys"]) { + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { - log.warn "module '$key': workflow argument '$nam' will be deprecated in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + log.warn "module '$key': workflow argument '$nam' will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." } } // check fromState - workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, thisConfig) + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) // check toState - workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, thisConfig) + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) // return output return workflowArgs @@ -118,7 +117,7 @@ def _processFromState(fromState, key_, config_) { assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" def fromStateMap = fromState.clone() - def requiredInputNames = thisConfig.functionality.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + def requiredInputNames = meta.config.functionality.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} // turn the map into a closure to be used later on fromState = { it -> def state = it[1] diff --git a/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf b/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf index 84631798e..28c1c46f2 100644 --- a/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf +++ b/src/main/resources/io/viash/platforms/nextflow/workflowFactory/workflowFactory.nf @@ -6,16 +6,16 @@ def _debug(workflowArgs, debugKey) { } } -// depends on: thisConfig, innerWorkflowFactory -def workflowFactory(Map args) { - def workflowArgs = processWorkflowArgs(args) +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) def key_ = workflowArgs["key"] workflow workflowInstance { take: input_ main: - mid1_ = input_ + chModified = input_ | checkUniqueIds([:]) | _debug(workflowArgs, "input") | map { tuple -> @@ -52,7 +52,7 @@ def workflowFactory(Map args) { // match file to input file if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { - def inputFiles = thisConfig.functionality.allArguments + def inputFiles = meta.config.functionality.allArguments .findAll { it.type == "file" && it.direction == "input" } assert inputFiles.size() == 1 : @@ -100,24 +100,36 @@ def workflowFactory(Map args) { } if (workflowArgs.filter) { - mid2_ = mid1_ + chModifiedFiltered = chModified | filter{workflowArgs.filter(it)} } else { - mid2_ = mid1_ + chModifiedFiltered = chModified + } + + if (workflowArgs.runIf) { + runIfBranch = chModifiedFiltered.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModifiedFiltered + chPassthrough = Channel.empty() } if (workflowArgs.fromState) { - mid3_ = mid2_ + chArgs = chRun | map{ def new_data = workflowArgs.fromState(it.take(2)) [it[0], new_data] } } else { - mid3_ = mid2_ + chArgs = chRun } // fill in defaults - mid4_ = mid3_ + chArgsWithDefaults = chArgs | map { tuple -> def id_ = tuple[0] def data_ = tuple[1] @@ -125,12 +137,12 @@ def workflowFactory(Map args) { // TODO: could move fromState to here // fetch default params from functionality - def defaultArgs = thisConfig.functionality.allArguments + def defaultArgs = meta.config.functionality.allArguments .findAll { it.containsKey("default") } .collectEntries { [ it.plainName, it.default ] } // fetch overrides in params - def paramArgs = thisConfig.functionality.allArguments + def paramArgs = meta.config.functionality.allArguments .findAll { par -> def argKey = key_ + "__" + par.plainName params.containsKey(argKey) @@ -138,7 +150,7 @@ def workflowFactory(Map args) { .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } // fetch overrides in data - def dataArgs = thisConfig.functionality.allArguments + def dataArgs = meta.config.functionality.allArguments .findAll { data_.containsKey(it.plainName) } .collectEntries { [ it.plainName, data_[it.plainName] ] } @@ -149,14 +161,14 @@ def workflowFactory(Map args) { combinedArgs .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} - combinedArgs = processInputs(combinedArgs, thisConfig, id_, key_) + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) [id_, combinedArgs] + tuple.drop(2) } // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - out0_ = mid4_ + chInitialOutput = chArgsWithDefaults | _debug(workflowArgs, "processed") // run workflow | innerWorkflowFactory(workflowArgs) @@ -175,7 +187,7 @@ def workflowFactory(Map args) { // remove metadata output_ = output_.findAll{k, v -> k != "_meta"} - output_ = processOutputs(output_, thisConfig, id_, key_) + output_ = _processOutputValues(output_, meta.config, id_, key_) if (workflowArgs.auto.simplifyOutput && output_.size() == 1) { output_ = output_.values()[0] @@ -183,11 +195,11 @@ def workflowFactory(Map args) { [meta_.join_id, meta_, id_, output_] } - // | view{"out0_: ${it.take(3)}"} + // | view{"chInitialOutput: ${it.take(3)}"} // TODO: this join will fail if the keys changed during the innerWorkflowFactory // join the output [join_id, meta, id, output] with the previous state [id, state, ...] - out1_ = safeJoin(out0_, mid2_, key_) + chNewState = safeJoin(chInitialOutput, chModifiedFiltered, key_) // input tuple format: [join_id, meta, id, output, prev_state, ...] // output tuple format: [join_id, meta, id, new_state, ...] | map{ tup -> @@ -196,44 +208,44 @@ def workflowFactory(Map args) { } if (workflowArgs.auto.publish == "state") { - out1pub_ = out1_ + chPublish = chNewState // input tuple format: [join_id, meta, id, new_state, ...] // output tuple format: [join_id, meta, id, new_state] | map{ tup -> tup.take(4) } - safeJoin(out1pub_, mid4_, key_) + safeJoin(chPublish, chArgsWithDefaults, key_) // input tuple format: [join_id, meta, id, new_state, orig_state, ...] // output tuple format: [id, new_state, orig_state] | map { tup -> tup.drop(2).take(3) } - | publishStatesByConfig(key: key_, config: thisConfig) + | publishStatesByConfig(key: key_, config: meta.config) } // remove join_id and meta - out2_ = out1_ + chReturn = chNewState | map { tup -> // input tuple format: [join_id, meta, id, new_state, ...] // output tuple format: [id, new_state, ...] tup.drop(2) } | _debug(workflowArgs, "output") + | concat(chPassthrough) - out2_ - - emit: out2_ + emit: chReturn } def wf = workflowInstance.cloneWithName(key_) // add factory function wf.metaClass.run = { runArgs -> - workflowFactory(runArgs) + // TODO: merge workflowArgs with runArgs + workflowFactory(runArgs, workflowArgs, meta) } // add config to module for later introspection - wf.metaClass.config = thisConfig + wf.metaClass.config = meta.config return wf } diff --git a/src/main/scala/io/viash/cli/CLIConf.scala b/src/main/scala/io/viash/cli/CLIConf.scala index 411b5b78e..950d9b1b5 100644 --- a/src/main/scala/io/viash/cli/CLIConf.scala +++ b/src/main/scala/io/viash/cli/CLIConf.scala @@ -390,7 +390,7 @@ class CLIConf(arguments: Seq[String]) extends ScallopConf(arguments) with Loggin short = Some('a'), default = Some(false), descr = - """Fills in the {platform} and {output} field by applying each platform to the + """Fills in the {platform} and {output} field by applying each platform to the |config separately. Note that this results in the provided command being applied |once for every platform that matches the --platform regex.""".stripMargin ) diff --git a/src/main/scala/io/viash/helpers/DependencyResolver.scala b/src/main/scala/io/viash/helpers/DependencyResolver.scala index 47f76baec..24659f18c 100644 --- a/src/main/scala/io/viash/helpers/DependencyResolver.scala +++ b/src/main/scala/io/viash/helpers/DependencyResolver.scala @@ -150,12 +150,17 @@ object DependencyResolver { config.map{ c => val path = c.info.get.config // fill in the location of the executable where it will be located - val executableName = platformId match { - case Some("nextflow") => "main.nf" - case _ => c.functionality.name + // TODO: it would be better if this was already filled in somewhere else + val executable = platformId.map{ pid => + val executableName = pid match { + case "nextflow" => "main.nf" + case _ => c.functionality.name + } + Paths.get(ViashNamespace.targetOutputPath("", pid, c.functionality.namespace, c.functionality.name), executableName).toString() } - val executable = Paths.get(ViashNamespace.targetOutputPath("", platformId.get, c.functionality.namespace, c.functionality.name), executableName).toString() - val info = c.info.get.copy(executable = Some(executable)) + val info = c.info.get.copy( + executable = executable + ) // Convert case class to map, do some extra conversions of Options while we're at it val map = (info.productElementNames zip info.productIterator).map{ case (k, s: String) => (k, s) @@ -163,7 +168,10 @@ object DependencyResolver { case (k, None) => (k, "") }.toMap // Add the functionality name and namespace to it - val map2 = Map(("functionalityName" -> c.functionality.name), ("functionalityNamespace" -> c.functionality.namespace.getOrElse(""))) + val map2 = Map( + ("functionalityName" -> c.functionality.name), + ("functionalityNamespace" -> c.functionality.namespace.getOrElse("")) + ) (path, map ++ map2) } } diff --git a/src/main/scala/io/viash/platforms/NextflowPlatform.scala b/src/main/scala/io/viash/platforms/NextflowPlatform.scala index f23c725dc..2d3202c4f 100644 --- a/src/main/scala/io/viash/platforms/NextflowPlatform.scala +++ b/src/main/scala/io/viash/platforms/NextflowPlatform.scala @@ -224,7 +224,7 @@ case class NextflowPlatform( val innerWorkflowFactory = mainScript match { // if mainscript is a nextflow workflow case scr: NextflowScript => - s"""// user-provided workflow + s"""// user-provided Nextflow code |${scr.readWithoutInjection.get.split("\n").mkString("\n|")} | |// inner workflow hook @@ -233,48 +233,50 @@ case class NextflowPlatform( |}""".stripMargin // else if it is a vdsl3 module case _ => - s"""// process script - |thisScript = ${NextflowHelper.generateScriptStr(config)} - | - |// inner workflow hook + s"""// inner workflow hook |def innerWorkflowFactory(args) { - | return vdsl3WorkflowFactory(args) + | def rawScript = ${NextflowHelper.generateScriptStr(config)} + | + | return vdsl3WorkflowFactory(args, meta, rawScript) |} | |""".stripMargin + NextflowHelper.vdsl3Helper } - s"""${NextflowHelper.generateHeader(config)} + NextflowHelper.generateHeader(config) + "\n\n" + + NextflowHelper.workflowHelper + + s""" | |nextflow.enable.dsl=2 | |// START COMPONENT-SPECIFIC CODE | - |// retrieve resourcesDir here to make sure the correct path is found - |resourcesDir = moduleDir.normalize() + |// create meta object + |meta = [ + | "resources_dir": moduleDir.normalize(), + | "config": ${NextflowHelper.generateConfigStr(config)} + |] | - |// component metadata - |thisConfig = ${NextflowHelper.generateConfigStr(config)} + |// resolve dependencies dependencies (if any) + |${NextflowHelper.renderDependencies(config).split("\n").mkString("\n|")} | |// inner workflow |${innerWorkflowFactory.split("\n").mkString("\n|")} | - |// component settings - |thisDefaultWorkflowArgs = ${NextflowHelper.generateDefaultWorkflowArgs(config, directivesToJson, auto, debug)} - | - |${NextflowHelper.renderDependencies(config).split("\n").mkString("\n|")} + |// defaults + |meta["defaults"] = ${NextflowHelper.generateDefaultWorkflowArgs(config, directivesToJson, auto, debug)} | |// initialise default workflow - |myWfInstance = workflowFactory([:]) + |meta["workflow"] = workflowFactory([key: meta.config.functionality.name], meta.defaults, meta) | |// add workflow to environment - |nextflow.script.ScriptMeta.current().addDefinition(myWfInstance) + |nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) | |// anonymous workflow for running this module as a standalone |workflow { | // add id argument if it's not already in the config - | if (thisConfig.functionality.arguments.every{it.plainName != "id"}) { + | if (meta.config.functionality.arguments.every{it.plainName != "id"}) { | def idArg = [ | 'name': '--id', | 'required': false, @@ -282,25 +284,23 @@ case class NextflowPlatform( | 'description': 'A unique id for every entry.', | 'multiple': false | ] - | thisConfig.functionality.arguments.add(0, idArg) - | thisConfig = processConfig(thisConfig) + | meta.config.functionality.arguments.add(0, idArg) + | meta.config = processConfig(meta.config) | } | if (!params.containsKey("id")) { | params.id = "run" | } | - | helpMessage(thisConfig) + | helpMessage(meta.config) | - | channelFromParams(params, thisConfig) - | | myWfInstance.run( + | channelFromParams(params, meta.config) + | | meta.workflow.run( | auto: [ publish: "state" ] | ) |} | |// END COMPONENT-SPECIFIC CODE - | - |""".stripMargin + - NextflowHelper.workflowHelper + |""".stripMargin } } diff --git a/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala b/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala index d0b57808c..4e4126ff3 100644 --- a/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala +++ b/src/main/scala/io/viash/platforms/nextflow/NextflowHelper.scala @@ -96,11 +96,14 @@ object NextflowHelper { s"""[ | // key to be used to trace the process and determine output names - | key: thisConfig.functionality.name, + | key: null, + | | // fixed arguments to be passed to script | args: [:], + | | // default directives | directives: readJsonBlob('''${jsonPrinter.print(dirJson2)}'''), + | | // auto settings | auto: readJsonBlob('''${jsonPrinter.print(autoJson)}'''), | @@ -124,6 +127,11 @@ object NextflowHelper { | // Example: `{ tup -> tup[0] == "foo" }` | filter: null, | + | // Choose whether or not to run the component on the tuple if the condition is true. + | // Otherwise, the tuple will be passed through. + | // Example: `{ tup -> tup[0] != "skip_this" }` + | runIf: null, + | | // Rename keys in the data field of the tuple (i.e. the second element) | // Will likely be deprecated in favour of `fromState`. | // Example: `[ "new_key": "old_key" ]` @@ -197,9 +205,9 @@ object NextflowHelper { // can we use suboutputpath here? //val dependencyPath = Paths.get(dependency.subOutputPath.get) val relativePath = parentPath.relativize(dependencyPath) - s"\"$$resourcesDir/$relativePath\"" + s"\"$${meta.resources_dir}/$relativePath\"" } else { - s"\"$$rootDir/dependencies/${dependency.subOutputPath.get}/main.nf\"" + s"\"$${meta.root_dir}/dependencies/${dependency.subOutputPath.get}/main.nf\"" } s"include { $depName$aliasStr } from ${source}" @@ -217,9 +225,7 @@ object NextflowHelper { return "" } - s""" - |// import dependencies - |rootDir = getRootDir() + s"""meta["root_dir"] = getRootDir() |${depStrs.mkString("\n|")} |""".stripMargin } diff --git a/src/main/scala/io/viash/schemas/AutoComplete.scala b/src/main/scala/io/viash/schemas/AutoComplete.scala index 73e90ebe9..3056b50ab 100644 --- a/src/main/scala/io/viash/schemas/AutoComplete.scala +++ b/src/main/scala/io/viash/schemas/AutoComplete.scala @@ -22,7 +22,7 @@ import io.viash.cli._ object AutoCompleteBash { def commandArguments(cmd: RegisteredCommand): String = { val (opts, trailOpts) = cmd.opts.partition(_.optType != "trailArgs") - val optNames = opts.map(_.name) ++ Seq("help") + val optNames = opts.filter(!_.hidden).map(_.name) ++ Seq("help") val cmdName = cmd.name trailOpts match { @@ -127,10 +127,11 @@ object AutoCompleteZsh { def getCleanDescr(opt: RegisteredOpt): String = { removeMarkup(opt.descr) .replaceAll("([\\[\\]\"])", "\\\\$1") // escape square brackets and quotes + .replaceAll("\n", " ") // remove newlines } val (opts, trailOpts) = cmd.opts.partition(_.optType != "trailArgs") - val cmdArgs = opts.map(o => + val cmdArgs = opts.filter(!_.hidden).map(o => if (o.short.isEmpty) { s""""--${o.name}[${getCleanDescr(o)}]"""" } else { @@ -145,8 +146,10 @@ object AutoCompleteZsh { | local -a cmd_args | cmd_args=( | ${cmdArgs.mkString("\n| ")} + | '(-h --help)'{-h,--help}'[Show help message]' + | ': :' | ) - | _arguments $$cmd_args $$_viash_help $$_viash_id_comp + | _arguments $$cmd_args | ;; |""".stripMargin case _ => @@ -155,8 +158,10 @@ object AutoCompleteZsh { | local -a cmd_args | cmd_args=( | ${cmdArgs.mkString("\n| ")} + | '(-h --help)'{-h,--help}'[Show help message]' + | ': :' | ) - | _arguments $$cmd_args $$_viash_help $$_viash_id_comp + | _arguments $$cmd_args | else | _files | fi @@ -182,7 +187,9 @@ object AutoCompleteZsh { | | if [[ CURRENT -eq 3 ]]; then | if [[ $${lastParam} == -* ]]; then - | _arguments $$_viash_help $$_viash_id_comp + | _arguments \\ + | '(-h --help)'{-h,--help}'[Show help message]' \\ + | ': :' | else | _describe -t commands "viash subcommands" ${cmdName}_commands | fi @@ -212,12 +219,6 @@ object AutoCompleteZsh { s"""#compdef viash - | - |local -a _viash_id_comp - |_viash_id_comp=('1: :->id_comp') - | - |local -a _viash_help - |_viash_help=('(-h --help)'{-h,--help}'[Show help message]') | |_viash_top_commands() { | local -a top_commands @@ -225,10 +226,13 @@ object AutoCompleteZsh { | ${topCmds.mkString("\n| ")} | ) | - | _arguments \\ - | '(-v --version)'{-v,--version}'[Show verson of this program]' \\ - | $$_viash_help \\ - | $$_viash_id_comp + | local -a cmd_args + | cmd_args=( + | '(-v --version)'{-v,--version}'[Show verson of this program]' + | '(-h --help)'{-h,--help}'[Show help message]' + | ': :' + | ) + | _arguments $$cmd_args | | _describe -t commands "viash subcommands" top_commands |} diff --git a/src/test/resources/testnextflowvdsl3/_viash.yaml b/src/test/resources/testnextflowvdsl3/_viash.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/filter_runif/config.vsh.yaml b/src/test/resources/testnextflowvdsl3/src/test_wfs/filter_runif/config.vsh.yaml new file mode 100644 index 000000000..c82db5ee3 --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/filter_runif/config.vsh.yaml @@ -0,0 +1,14 @@ +functionality: + name: filter_runif + namespace: test_wfs + resources: + - type: nextflow_script + path: main.nf + entrypoint: base + - path: /resources + dependencies: + - name: step1 + - name: step2 + - name: step3 +platforms: + - type: nextflow \ No newline at end of file diff --git a/src/test/resources/testnextflowvdsl3/src/test_wfs/filter_runif/main.nf b/src/test/resources/testnextflowvdsl3/src/test_wfs/filter_runif/main.nf new file mode 100644 index 000000000..27c1e2834 --- /dev/null +++ b/src/test/resources/testnextflowvdsl3/src/test_wfs/filter_runif/main.nf @@ -0,0 +1,42 @@ +lines3 = meta.resources_dir.resolve("resources/lines3.txt") +lines5 = meta.resources_dir.resolve("resources/lines5.txt") + +workflow base { + take: input_ch + main: + + Channel.fromList([ + ["one", [input: [lines3]]], + ["two", [input: [lines3, lines5]]], + ["three", [input: [lines5]]] + ]) + | step1.run( + filter: { id, data -> id != "three" }, + runIf: { id, data -> data.input.size() == 2 } + ) + | view{"output: $it"} + | toList() + | view { tup_list -> + assert tup_list.size() == 2 : "output channel should contain 2 events" + + def tup0 = tup_list[0] + assert tup0.size() == 2 : "outputs should contain two elements; [id, output]" + + // check id + assert tup0[0] == "one" : "id should be one" + assert tup1[0] == "two" : "id should be two" + + // check data + assert tup[0].containsKey("input") : "data should contain key input" + assert tup[0].input.size() == 1 : "data should contain 1 file" + assert tup[0].input[0].name == "lines3.txt" : "input should contain lines3.txt" + + assert tup[1].containsKey("output") : "data should contain key output" + assert tup[1].output == 1 : "data should contain 1 file" + assert tup[1].output.name == "lines3.txt" : "input should contain lines3.txt" + + "" + } + emit: + Channel.empty() +} \ No newline at end of file diff --git a/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf b/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf index 0ea431e00..10c560b55 100644 --- a/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf +++ b/src/test/resources/testnextflowvdsl3/workflows/pipeline1/main.nf @@ -11,12 +11,13 @@ include { step2 } from "$targetDir/step2/main.nf" // ["input": List[File]] -> File include { step3 } from "$targetDir/step3/main.nf" +lines3 = file("${params.rootDir}/resources/lines3.txt") +lines5 = file("${params.rootDir}/resources/lines5.txt") + def channelValue = Channel.value([ "foo", // id - [ // data - "input": file("${params.rootDir}/resources/lines*.txt") - ], - file("${params.rootDir}/resources/lines3.txt") // pass-through + ["input": [lines3, lines5]], + lines3 // pass-through ]) workflow base { @@ -179,4 +180,41 @@ workflow test_fromstate_tostate_arguments { // return something to print "DEBUG4: $output" } +} + + +workflow test_filter_runif_arguments { + Channel.fromList([ + ["one", [input: [lines3]]], + ["two", [input: [lines3, lines5]]], + ["three", [input: [lines5]]] + ]) + | step1.run( + filter: { id, data -> id != "three" }, + runIf: { id, data -> data.input.size() == 2 } + ) + | toSortedList( { a, b -> a[0] <=> b[0] } ) + | view { tups -> + assert tups.size() == 2 : "output channel should contain 1 event" + + def tup0 = tups[0] + def tup1 = tups[1] + assert tup0.size() == 2 : "outputs should contain two elements; [id, output]" + assert tup1.size() == 2 : "outputs should contain two elements; [id, output]" + + // check id + assert tup0[0] == "one" : "id should be one" + assert tup1[0] == "two" : "id should be two" + + // check data + assert tup0[1].containsKey("input") : "data should contain key input" + assert tup0[1].input.size() == 1 : "data should contain 1 file" + assert tup0[1].input[0].name == "lines3.txt" : "data should contain lines3.txt. Found: ${tup0[1]}" + + assert tup1[1].containsKey("output") : "data should contain key output" + assert tup1[1].output instanceof Path : "data should contain 1 file" + assert tup1[1].output.name == "two.step1.output.txt" : "data should contain two.step1.output.txt. Found: ${tup1[1]}" + + "" + } } \ No newline at end of file diff --git a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala index 191d32eeb..0fed35ec5 100644 --- a/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala +++ b/src/test/scala/io/viash/e2e/test/MainTestDockerSuite.scala @@ -95,11 +95,28 @@ class MainTestDockerSuite extends AnyFunSuite with BeforeAndAfterAll with Parall "--setup", "cb", "--keep", "false" ) - assert(regexBuildCache.findFirstIn(testTextCaching).isDefined, "Expected to find caching.") + + // retry once if it failed + val testTextCachingWithRetry = + if (regexBuildCache.findFirstIn(testTextCaching).isDefined) { + testTextCaching + } else { + checkTempDirAndRemove(testTextCaching, false) + + TestHelper.testMain( + "test", + "-p", "docker", + configFile, + "--setup", "cb", + "--keep", "false" + ) + } + + assert(regexBuildCache.findFirstIn(testTextCachingWithRetry).isDefined, "Expected to find caching.") + checkTempDirAndRemove(testText, false) - checkTempDirAndRemove(testTextCaching, false) checkTempDirAndRemove(testTextNoCaching, false) - + checkTempDirAndRemove(testTextCachingWithRetry, false) } test("Verify base config derivation", NativeTest) { diff --git a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala index 65bbc48b5..2ae8b92c7 100644 --- a/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/NextflowScriptTest.scala @@ -57,17 +57,6 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { ) assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") - - // TODO: add back checks? - - // outputFileMatchChecker(stdOut, "DEBUG6", "^11 .*$") - - // // check whether step3's debug printing was triggered - // outputFileMatchChecker(stdOut, "process 'step3[^']*' output tuple", "^11 .*$") - - // // check whether step2's debug printing was not triggered - // val lines2 = stdOut.split("\n").find(_.contains("process 'step2' output tuple")) - // assert(!lines2.isDefined) } // TODO: use TestHelper.testMainWithStdErr instead of NextflowTestHelper.run; i.e. viash test @@ -86,6 +75,20 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") } + // why is this not working? + + // test("Test filter/runIf", DockerTest, NextflowTest) { + // val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + // mainScript = "target/nextflow/test_wfs/filter_runif/main.nf", + // args = List( + // "--publish_dir", "output" + // ), + // cwd = tempFolFile + // ) + + // assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + // } + test("Check whether --help is same as Viash's --help", NextflowTest) { // except that WorkflowHelper.nf will not print alternatives, and @@ -100,26 +103,6 @@ class NextflowScriptTest extends AnyFunSuite with BeforeAndAfterAll { ) assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut1\nStd error:\n$stdErr1") - - // // explicitly remove defaults set by output files - // // these defaults make sense in nextflow but not in viash - // val correctedStdOut1 = stdOut1.replaceAll(" default: \\$id\\.\\$key\\.[^\n]*\n", "") - // // explicitly remove global arguments - // // these arguments make sense in nextflow but not in viash - // import java.util.regex.Pattern - // val regex = Pattern.compile("\nNextflow input-output arguments:.*", Pattern.DOTALL) - // val correctedStdOut2 = regex.matcher(correctedStdOut1).replaceAll("") - - // // run Viash's --help - // val (stdOut2, stdErr2, exitCode2) = TestHelper.testMainWithStdErr( - // "run", srcPath + "/wf/config.vsh.yaml", - // "--", "--help" - // ) - - // assert(exitCode2 == 0) - - // // check if they are the same - // assert(correctedStdOut2 == stdOut2) } diff --git a/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala b/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala index c63ba3d05..ee0bb0b26 100644 --- a/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala +++ b/src/test/scala/io/viash/platforms/nextflow/Vdsl3ModuleTest.scala @@ -86,6 +86,18 @@ class Vdsl3ModuleTest extends AnyFunSuite with BeforeAndAfterAll { assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") } + test("Test filter/runIf arguments", DockerTest, NextflowTest) { + + val (exitCode, stdOut, stdErr) = NextflowTestHelper.run( + mainScript = "workflows/pipeline1/main.nf", + entry = Some("test_filter_runif_arguments"), + args = List("--publish_dir", "output"), + cwd = tempFolFile + ) + + assert(exitCode == 0, s"\nexit code was $exitCode\nStd output:\n$stdOut\nStd error:\n$stdErr") + } + test("Check whether --help is same as Viash's --help", NextflowTest) { // except that WorkflowHelper.nf will not print alternatives, and // will always prefix argument names with -- (so --foo, not -f or foo).