Merge pull request #559 from viash-io/develop

Viash 0.8.0-RC4

CHANGELOG.md

@@ -1,3 +1,19 @@
+# Viash 0.8.0-RC4 (2023-10-10): Refactor Nextflow helper functions
+
+## MAJOR CHANGES
+
+* `NextflowPlatform`: Refactoring of helper functions (PR #557).
+  - Cleaned up `processConfig()`: Removed support for `functionality.inputs` and `functionality.outputs`
+  - Cleaned up `processConfig()`: Removed support for `.functionality.argument_groups[].argument` containing a list of argument ids as opposed to the arguments themselves.
+  - Rewrote `--param_list` parser.
+  - Removed unused function `applyConfig()` and `applyConfigToOneParamSet()`.
+  - Refactored `channelFromParams()` to make use of new helper functions.
+  - Removed deprecated `paramsToChannel()`, `paramsToList()`, `viashChannel()`.
+  - Deprecated `preprocessInputs()` -- use the wrapped Viash Nextflow functionality instead.
+  - Refactored `preprocessInputs()` to make use of new helper functions.
+  - Reprecated run arguments `map`, `mapData`, `mapPassthrough`, `renameKeys`.
+
+
 # Viash 0.8.0-RC3 (2023-10-07): More bugfixes, more Nextflow functionality
 
 This release contains more bugfixes related to the Nextflow code generation functionality.

build.sbt

@@ -1,6 +1,6 @@
 name := "viash"
 
-version := "0.8.0-RC3"
+version := "0.8.0-RC4"
 
 scalaVersion := "2.13.10"
 

src/main/resources/io/viash/platforms/nextflow/DataflowHelper.nf

@@ -4,7 +4,7 @@ dataflowHelperDeprecationWarningPrinted = false
 def dataflowHelperDeprecationWarning() {
   if (!dataflowHelperDeprecationWarningPrinted) {
     dataflowHelperDeprecationWarningPrinted = true
-    System.err.println("Warning: the functions in the DataflowHelper.nf (setWorkflowArguments, getWorkflowArguments) are set to be deprecated Viash 0.9.0.")
+    System.err.println("Warning: the functions in the DataflowHelper.nf (setWorkflowArguments, getWorkflowArguments) are deprecated and will be removed in Viash 0.9.0.")
   }
 }
 

src/main/resources/io/viash/platforms/nextflow/arguments/_checkArgumentType.nf

@@ -1,64 +1,155 @@
+class UnexpectedArgumentTypeException extends Exception {
+  String errorIdentifier
+  String stage
+  String plainName
+  String expectedClass
+  String foundClass
+
+  // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""}
+  UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) {
+    super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " +
+      "Expected type: ${expectedClass}. Found type: ${foundClass}")
+    this.errorIdentifier = errorIdentifier
+    this.stage = stage
+    this.plainName = plainName
+    this.expectedClass = expectedClass
+    this.foundClass = foundClass
+  }
+}
 
-def _checkArgumentType(String stage, Map par, Object value, String id, String key) {
+/**
+  * Checks if the given value is of the expected type. If not, an exception is thrown.
+  *
+  * @param stage The stage of the argument (input or output)
+  * @param par The parameter definition
+  * @param value The value to check
+  * @param errorIdentifier The identifier to use in the error message
+  * @return The value, if it is of the expected type
+  * @throws UnexpectedArgumentTypeException If the value is not of the expected type
+*/
+def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) {
   // expectedClass will only be != null if value is not of the expected type
   def expectedClass = null
+  def foundClass = null
+
+  // todo: split if need be
 
   if (!par.required && value == null) {
     expectedClass = null
   } else if (par.multiple) {
-    if (value instanceof List) {
-      try {
-        value = value.collect { listVal ->
-          _checkArgumentType(stage, par + [multiple: false], listVal, id, key)
-        }
-      } catch (Exception e) {
-        expectedClass = "List[${par.type}]"
+    if (value !instanceof Collection) {
+      value = [value]
+    }
+
+    // split strings
+    value = value.collectMany{ val ->
+      if (val instanceof String) {
+        // collect() to ensure that the result is a List and not simply an array
+        val.split(par.multiple_sep).collect()
+      } else {
+        [val]
+      }
+    }
+
+    // process globs
+    if (par.type == "file" && par.direction == "input") {
+      value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten()
+    }
+
+    // check types of elements in list
+    try {
+      value = value.collect { listVal ->
+        _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier)
       }
-    } else {
-      expectedClass = "List[${par.type}]"
+    } catch (UnexpectedArgumentTypeException e) {
+      expectedClass = "List[${e.expectedClass}]"
+      foundClass = "List[${e.foundClass}]"
     }
   } else if (par.type == "string") {
+    // cast to string if need be
     if (value instanceof GString) {
       value = value.toString()
     }
     expectedClass = value instanceof String ? null : "String"
   } else if (par.type == "integer") {
+    // cast to integer if need be
+    if (value instanceof String) {
+      try {
+        value = value.toInteger()
+      } catch (NumberFormatException e) {
+        // do nothing
+      }
+    }
+    if (value instanceof java.math.BigInteger) {
+      value = value.intValue()
+    }
     expectedClass = value instanceof Integer ? null : "Integer"
   } else if (par.type == "long") {
+    // cast to long if need be
+    if (value instanceof String) {
+      try {
+        value = value.toLong()
+      } catch (NumberFormatException e) {
+        // do nothing
+      }
+    }
     if (value instanceof Integer) {
       value = value.toLong()
     }
     expectedClass = value instanceof Long ? null : "Long"
   } else if (par.type == "double") {
+    // cast to double if need be
+    if (value instanceof String) {
+      try {
+        value = value.toDouble()
+      } catch (NumberFormatException e) {
+        // do nothing
+      }
+    }
     if (value instanceof java.math.BigDecimal) {
       value = value.doubleValue()
     }
-    if (value instanceof Float || value instanceof Integer || value instanceof Long) {
+    if (value instanceof Float) {
       value = value.toDouble()
     }
     expectedClass = value instanceof Double ? null : "Double"
   } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") {
-    expectedClass = value instanceof Boolean ? null : "Boolean"
-  } else if (par.type == "file") {
-    if (stage == "output" || par.direction == "input") {
-      if (value instanceof File) {
-        value = value.toPath()
+    // cast to boolean if need ben
+    if (value instanceof String) {
+      def valueLower = value.toLowerCase()
+      if (valueLower == "true") {
+        value = true
+      } else if (valueLower == "false") {
+        value = false
       }
-      expectedClass = value instanceof Path ? null : "Path"
-    } else { // stage == "input" && par.direction == "output"
-      if (value instanceof GString) {
-        value = value.toString()
-      }
-      expectedClass = value instanceof String ? null : "String"
     }
+    expectedClass = value instanceof Boolean ? null : "Boolean"
+  } else if (par.type == "file" && (par.direction == "input" || stage == "output")) {
+    // cast to path if need be
+    if (value instanceof String) {
+      value = file(value, hidden: true)
+    }
+    if (value instanceof File) {
+      value = value.toPath()
+    }
+    expectedClass = value instanceof Path ? null : "Path"
+  } else if (par.type == "file" && stage == "input" && par.direction == "output") {
+    // cast to string if need be
+    if (value instanceof GString) {
+      value = value.toString()
+    }
+    expectedClass = value instanceof String ? null : "String"
   } else {
+    // didn't find a match for par.type
     expectedClass = par.type
   }
 
   if (expectedClass != null) {
-    error "Error in module '${key}' id '${id}': ${stage} argument '${par.plainName}' has the wrong type. " +
-      "Expected type: ${expectedClass}. Found type: ${value.getClass()}"
+    if (foundClass == null) {
+      foundClass = value.getClass().getName()
+    }
+    throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass)
   }
 
   return value
-}
+}

src/main/resources/io/viash/platforms/nextflow/arguments/_processInputValues.nf

@@ -11,7 +11,7 @@ Map _processInputValues(Map inputs, Map config, String id, String key) {
       def par = config.functionality.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument"
 
-      value = _checkArgumentType("input", par, value, id, key)
+      value = _checkArgumentType("input", par, value, "in module '$key' id '$id'")
 
       [ name, value ]
     }

src/main/resources/io/viash/platforms/nextflow/arguments/_processOutputValues.nf

@@ -11,7 +11,7 @@ Map _processOutputValues(Map outputs, Map config, String id, String key) {
       def par = config.functionality.allArguments.find { it.plainName == name && it.direction == "output" }
       assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument"
 
-      value = _checkArgumentType("output", par, value, id, key)
+      value = _checkArgumentType("output", par, value, "in module '$key' id '$id'")
 
       [ name, value ]
     }

src/main/resources/io/viash/platforms/nextflow/channel/_parseParamList.nf

@@ -0,0 +1,107 @@
+/**
+  * Figure out the param list format based on the file extension
+  *
+  * @param param_list A String containing the path to the parameter list file.
+  *
+  * @return A String containing the format of the parameter list file.
+  */
+def _paramListGuessFormat(param_list) {
+  if (param_list !instanceof String) {
+    "asis"
+  } else if (param_list.endsWith(".csv")) {
+    "csv"
+  } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) {
+    "json"
+  } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) {
+    "yaml"
+  } else {
+    "yaml_blob"
+  }
+}
+
+
+/**
+  * Read the param list
+  * 
+  * @param param_list One of the following:
+  *   - A String containing the path to the parameter list file (csv, json or yaml),
+  *   - A yaml blob of a list of maps (yaml_blob),
+  *   - Or a groovy list of maps (asis).
+  * @param config A Map of the Viash configuration.
+  * 
+  * @return A List of Maps containing the parameters.
+  */
+def _parseParamList(param_list, Map config) {
+  // first determine format by extension
+  def paramListFormat = _paramListGuessFormat(param_list)
+
+  def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ?
+    file(param_list, hidden: true) :
+    null
+
+  // get the correct parser function for the detected params_list format
+  def paramSets = []
+  if (paramListFormat == "asis") {
+    paramSets = param_list
+  } else if (paramListFormat == "yaml_blob") {
+    paramSets = readYamlBlob(param_list)
+  } else if (paramListFormat == "yaml") {
+    paramSets = readYaml(paramListPath)
+  } else if (paramListFormat == "json") {
+    paramSets = readJson(paramListPath)
+  } else if (paramListFormat == "csv") {
+    paramSets = readCsv(paramListPath)
+  } else {
+    error "Format of provided --param_list not recognised.\n" +
+    "Found: '$paramListFormat'.\n" +
+    "Expected: a csv file, a json file, a yaml file,\n" +
+    "a yaml blob or a groovy list of maps."
+  }
+
+  // data checks
+  assert paramSets instanceof List: "--param_list should contain a list of maps"
+  for (value in paramSets) {
+    assert value instanceof Map: "--param_list should contain a list of maps"
+  }
+
+  // id is argument
+  def idIsArgument = config.functionality.allArguments.any{it.plainName == "id"}
+
+  // Reformat from List<Map> to List<Tuple2<String, Map>> by adding the ID as first element of a Tuple2
+  paramSets = paramSets.collect({ data ->
+    def id = data.id
+    if (!idIsArgument) {
+      data = data.findAll{k, v -> k != "id"}
+    }
+    [id, data]
+  })
+
+  // Split parameters with 'multiple: true'
+  paramSets = paramSets.collect({ id, data ->
+    data = _splitParams(data, config)
+    [id, data]
+  })
+
+  // The paths of input files inside a param_list file may have been specified relatively to the
+  // location of the param_list file. These paths must be made absolute.
+  if (paramListPath) {
+    paramSets = paramSets.collect({ id, data ->
+      def new_data = data.collectEntries{ parName, parValue ->
+        def par = config.functionality.allArguments.find{it.plainName == parName}
+        if (par && par.type == "file" && par.direction == "input") {
+          if (parValue instanceof Collection) {
+            parValue = parValue.collect{path ->
+              path instanceof String ? paramListPath.resolveSibling(path) : path
+            }
+          } else {
+            parValue = parValue instanceof String ? paramListPath.resolveSibling(parValue) : parValue
+          }
+        }
+        [parName, parValue]
+      }
+      [id, new_data]
+    })
+  }
+
+  return paramSets
+}