Merge pull request #86 from viash-io/develop

viash 0.5.3
viash-io · Sep 2, 2021 · 45a1f4c · 45a1f4c
2 parents 5acdd45 + 44dc967
commit 45a1f4c
Show file tree

Hide file tree

Showing 23 changed files with 208 additions and 63 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,18 @@
+# viash 0.5.3
+
+## NEW FEATURES
+
+* Similar to `par`, each script now also has a `meta` list. `meta` contains meta information about the component
+  or the execution thereof. It currently has the following fields:
+  - `meta["resources_dir"]`: Path to the directory containing the resources
+  - `meta["functionality_name"]`: Name of the component
+
+* `NextFlowPlatform`: Export `VIASH_TEMP` environment variable. 
+
+## BUG FIXES
+
+* `NextFlowPlatform`: Fix output formatting when `separate_multiple_outputs` is `false`.
+
 # viash 0.5.2
 
 ## MINOR CHANGES

diff --git a/build.sbt b/build.sbt
@@ -1,6 +1,6 @@
 name := "viash"
 
-version := "0.5.2"
+version := "0.5.3"
 
 scalaVersion := "2.12.10"
 

diff --git a/src/main/scala/com/dataintuitive/viash/functionality/resources/BashScript.scala b/src/main/scala/com/dataintuitive/viash/functionality/resources/BashScript.scala
@@ -19,6 +19,7 @@ package com.dataintuitive.viash.functionality.resources
 
 import com.dataintuitive.viash.functionality._
 import com.dataintuitive.viash.functionality.dataobjects._
+import com.dataintuitive.viash.wrapper.BashWrapper
 
 import java.net.URI
 
@@ -38,11 +39,14 @@ case class BashScript(
   def generatePlaceholder(functionality: Functionality): String = {
     val params = functionality.arguments.filter(d => d.direction == Input || d.isInstanceOf[FileObject])
 
-    val par_set = params.map { par =>
+    val parSet = params.map { par =>
       s"""${par.par}='$$${par.VIASH_PAR}'"""
     }
-    s"""${par_set.mkString("\n")}
-       |
+    val metaSet = BashWrapper.metaFields.map{ case (env_name, script_name) =>
+      s"""meta_$script_name='$$$env_name'""".stripMargin
+    }
+    s"""${parSet.mkString("\n")}
+       |${metaSet.mkString("\n")}
        |resources_dir="$$VIASH_RESOURCES_DIR"
        |""".stripMargin
   }

diff --git a/src/main/scala/com/dataintuitive/viash/functionality/resources/CSharpScript.scala b/src/main/scala/com/dataintuitive/viash/functionality/resources/CSharpScript.scala
@@ -19,6 +19,7 @@ package com.dataintuitive.viash.functionality.resources
 
 import com.dataintuitive.viash.functionality._
 import com.dataintuitive.viash.functionality.dataobjects._
+import com.dataintuitive.viash.wrapper.BashWrapper
 
 import java.net.URI
 
@@ -83,10 +84,15 @@ case class CSharpScript(
 
       s"${par.plainName} = $setter"
     }
+    val metaSet = BashWrapper.metaFields.map{ case (env_name, script_name) =>
+      s"""$script_name = "$$$env_name""""
+    }
     s"""var par = new {
        |  ${parSet.mkString(",\n  ")}
        |};
-       |
+       |var meta = new {
+       |  ${metaSet.mkString(",\n  ")}
+       |};
        |var resources_dir = "$$VIASH_RESOURCES_DIR";
        |""".stripMargin
   }

diff --git a/src/main/scala/com/dataintuitive/viash/functionality/resources/JavaScriptScript.scala b/src/main/scala/com/dataintuitive/viash/functionality/resources/JavaScriptScript.scala
@@ -19,6 +19,7 @@ package com.dataintuitive.viash.functionality.resources
 
 import com.dataintuitive.viash.functionality._
 import com.dataintuitive.viash.functionality.dataobjects._
+import com.dataintuitive.viash.wrapper.BashWrapper
 
 import java.net.URI
 
@@ -38,7 +39,7 @@ case class JavaScriptScript(
   def generatePlaceholder(functionality: Functionality): String = {
     val params = functionality.arguments.filter(d => d.direction == Input || d.isInstanceOf[FileObject])
 
-    val par_set = params.map { par =>
+    val parSet = params.map { par =>
       val env_name = par.VIASH_PAR
 
       val parse = par match {
@@ -61,10 +62,15 @@ case class JavaScriptScript(
 
       s"""'${par.plainName}': $$VIASH_DOLLAR$$( if [ ! -z $${$env_name+x} ]; then echo "$parse"; else echo undefined; fi )"""
     }
+    val metaSet = BashWrapper.metaFields.map{ case (env_name, script_name) =>
+      s"""'$script_name': '$$$env_name'"""
+    }
     s"""let par = {
-       |  ${par_set.mkString(",\n  ")}
+       |  ${parSet.mkString(",\n  ")}
+       |};
+       |let meta = {
+       |  ${metaSet.mkString(",\n  ")}
        |};
-       |
        |let resources_dir = '$$VIASH_RESOURCES_DIR'
        |""".stripMargin
   }

diff --git a/src/main/scala/com/dataintuitive/viash/functionality/resources/PythonScript.scala b/src/main/scala/com/dataintuitive/viash/functionality/resources/PythonScript.scala
@@ -19,6 +19,7 @@ package com.dataintuitive.viash.functionality.resources
 
 import com.dataintuitive.viash.functionality._
 import com.dataintuitive.viash.functionality.dataobjects._
+import com.dataintuitive.viash.wrapper.BashWrapper
 
 import java.net.URI
 
@@ -38,7 +39,7 @@ case class PythonScript(
   def generatePlaceholder(functionality: Functionality): String = {
     val params = functionality.arguments.filter(d => d.direction == Input || d.isInstanceOf[FileObject])
 
-    val par_set = params.map { par =>
+    val parSet = params.map { par =>
       val env_name = par.VIASH_PAR
 
       val parse = par match {
@@ -61,8 +62,15 @@ case class PythonScript(
 
       s"""'${par.plainName}': $$VIASH_DOLLAR$$( if [ ! -z $${$env_name+x} ]; then echo "$parse"; else echo None; fi )"""
     }
+    val metaSet = BashWrapper.metaFields.map{ case (env_name, script_name) =>
+      s"""'$script_name': '$$$env_name'"""
+    }
+
     s"""par = {
-       |  ${par_set.mkString(",\n  ")}
+       |  ${parSet.mkString(",\n  ")}
+       |}
+       |meta = {
+       |  ${metaSet.mkString(",\n  ")}
        |}
        |
        |resources_dir = '$$VIASH_RESOURCES_DIR'

diff --git a/src/main/scala/com/dataintuitive/viash/functionality/resources/RScript.scala b/src/main/scala/com/dataintuitive/viash/functionality/resources/RScript.scala
@@ -19,6 +19,7 @@ package com.dataintuitive.viash.functionality.resources
 
 import com.dataintuitive.viash.functionality._
 import com.dataintuitive.viash.functionality.dataobjects._
+import com.dataintuitive.viash.wrapper.BashWrapper
 
 import java.net.URI
 
@@ -38,7 +39,7 @@ case class RScript(
   def generatePlaceholder(functionality: Functionality): String = {
     val params = functionality.arguments.filter(d => d.direction == Input || d.isInstanceOf[FileObject])
 
-    val par_set = params.map { par =>
+    val parSet = params.map { par =>
       val env_name = par.VIASH_PAR
 
       val parse = par match {
@@ -61,10 +62,21 @@ case class RScript(
 
       s""""${par.plainName}" = $$VIASH_DOLLAR$$( if [ ! -z $${$env_name+x} ]; then echo "$parse"; else echo NULL; fi )"""
     }
-    s"""par <- list(
-       |  ${par_set.mkString(",\n  ")}
+    val metaSet = BashWrapper.metaFields.map{ case (env_name, script_name) =>
+      s"""$script_name = "$$$env_name""""
+    }
+
+    s"""# get parameters from cli
+       |par <- list(
+       |  ${parSet.mkString(",\n  ")}
+       |)
+       |
+       |# get meta parameters
+       |meta <- list(
+       |  ${metaSet.mkString(",\n  ")}
        |)
        |
+       |# get resources dir
        |resources_dir = "$$VIASH_RESOURCES_DIR"
        |""".stripMargin
   }

diff --git a/src/main/scala/com/dataintuitive/viash/functionality/resources/ScalaScript.scala b/src/main/scala/com/dataintuitive/viash/functionality/resources/ScalaScript.scala
@@ -19,6 +19,7 @@ package com.dataintuitive.viash.functionality.resources
 
 import com.dataintuitive.viash.functionality._
 import com.dataintuitive.viash.functionality.dataobjects._
+import com.dataintuitive.viash.wrapper.BashWrapper
 
 import java.net.URI
 
@@ -38,7 +39,7 @@ case class ScalaScript(
   def generatePlaceholder(functionality: Functionality): String = {
     val params = functionality.arguments.filter(d => d.direction == Input || d.isInstanceOf[FileObject])
 
-    val classTypes = params.map { par =>
+    val parClassTypes = params.map { par =>
       val classType = par match {
         case o: BooleanObject if o.multiple => "List[Boolean]"
         case o: IntegerObject if o.multiple => "List[Integer]"
@@ -58,9 +59,7 @@ case class ScalaScript(
         case _: StringObject => "String"
       }
       par.plainName + ": " + classType
-
     }
-
     val parSet = params.map { par =>
       val env_name = par.VIASH_PAR
 
@@ -100,14 +99,27 @@ case class ScalaScript(
         case None => parse
       }
     }
+
+    val metaClassTypes = BashWrapper.metaFields.map { case (_, script_name) =>
+      script_name + ": String"
+    }
+    val metaSet = BashWrapper.metaFields.map { case (env_name, script_name) =>
+      s""""$$$env_name""""
+    }
     s"""case class ViashPar(
-       |  ${classTypes.mkString(",\n  ")}
+       |  ${parClassTypes.mkString(",\n  ")}
        |)
-       |
        |val par = ViashPar(
        |  ${parSet.mkString(",\n  ")}
        |)
        |
+       |case class ViashMeta(
+       |  ${metaClassTypes.mkString(",\n  ")}
+       |)
+       |val meta = ViashMeta(
+       |  ${metaSet.mkString(",\n  ")}
+       |)
+       |
        |val resources_dir = "$$VIASH_RESOURCES_DIR"
        |""".stripMargin
   }

diff --git a/src/main/scala/com/dataintuitive/viash/platforms/NextFlowPlatform.scala b/src/main/scala/com/dataintuitive/viash/platforms/NextFlowPlatform.scala
@@ -167,7 +167,7 @@ case class NextFlowPlatform(
      */
     val asNestedTuples: List[ConfigTuple] = List(
       "docker.enabled" → true,
-      "docker.runOptions" → "-i -v ${baseDir}:${baseDir}",
+      "def viash_temp = System.getenv(\"VIASH_TEMP\") ?: \"/tmp/\"\n  docker.runOptions" → "-i -v ${baseDir}:${baseDir} -v $viash_temp:$viash_temp",
       "process.container" → "dataintuitive/viash",
       "params" → NestedValue(
         namespacedParameters :::
@@ -461,11 +461,14 @@ case class NextFlowPlatform(
         |    STUB=1 $$cli
         |    \"\"\"
         |  script:
+        |    def viash_temp = System.getenv("VIASH_TEMP") ?: "/tmp/"
         |    if (params.test)
         |      \"\"\"
         |      # Some useful stuff
         |      export NUMBA_CACHE_DIR=/tmp/numba-cache
         |      # Running the pre-hook when necessary
+        |      # Pass viash temp dir
+        |      export VIASH_TEMP="$${viash_temp}"
         |      # Adding NXF's `$$moduleDir` to the path in order to resolve our own wrappers
         |      export PATH="./:$${moduleDir}:\\$$PATH"
         |      ./$${params.$fname.tests.testScript} | tee $$output
@@ -475,6 +478,8 @@ case class NextFlowPlatform(
         |      # Some useful stuff
         |      export NUMBA_CACHE_DIR=/tmp/numba-cache
         |      # Running the pre-hook when necessary
+        |      # Pass viash temp dir
+        |      export VIASH_TEMP="$${viash_temp}"
         |      # Adding NXF's `$$moduleDir` to the path in order to resolve our own wrappers
         |      export PATH="$${moduleDir}:\\$$PATH"
         |      $$cli
@@ -499,7 +504,7 @@ case class NextFlowPlatform(
         s"""  emit:
            |  result_.flatMap{ it ->
            |    (it[1].keySet().size() > 1)
-           |      ? it
+           |      ? [ it ]
            |      : it[1].collect{ k, el -> [ it[0], el, it[2] ] }
            |  }""".stripMargin
       }

diff --git a/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala b/src/main/scala/com/dataintuitive/viash/wrapper/BashWrapper.scala
@@ -24,6 +24,12 @@ import com.dataintuitive.viash.functionality.dataobjects._
 import com.dataintuitive.viash.helpers.{Bash, Format}
 
 object BashWrapper {
+  val metaFields: List[(String, String)] = {
+    List(
+      ("VIASH_META_FUNCTIONALITY_NAME", "functionality_name"),
+      ("VIASH_RESOURCES_DIR", "resources_dir")
+    )
+  }
 
   def nameAndVersion(functionality: Functionality): String = {
     functionality.name + functionality.version.map(" " + _).getOrElse(" <not versioned>")
@@ -179,6 +185,9 @@ object BashWrapper {
        |# find source folder of this component
        |$var_resources_dir=`ViashSourceDir $${BASH_SOURCE[0]}`
        |
+       |# define meta fields
+       |VIASH_META_FUNCTIONALITY_NAME="${functionality.name}"
+       |
        |${spaceCode(allMods.preParse)}
        |# initialise array
        |VIASH_POSITIONAL_ARGS=''

diff --git a/src/test/resources/testbash/code.sh b/src/test/resources/testbash/code.sh
@@ -12,6 +12,7 @@ par_output="output.txt"
 par_log="log.txt"
 par_optional="help"
 par_optional_with_default="me"
+meta_functionality_name="testbash"
 resources_dir="."
 # VIASH END
 
@@ -58,3 +59,6 @@ RESOURCE=`head -1 "$resources_dir/resource1.txt"`
 output "head of resource1: |$RESOURCE|"
 output "multiple: |$par_multiple|"
 output "multiple_pos: |$par_multiple_pos|"
+
+output "meta_functionality_name: |$meta_functionality_name|"
+output "meta_resources_dir: |$meta_resources_dir|"
diff --git a/src/test/resources/testbash/tests/check_outputs.sh b/src/test/resources/testbash/tests/check_outputs.sh
@@ -24,6 +24,8 @@ grep -q 'optional: |foo|' output.txt
 grep -q 'optional_with_default: |bar|' output.txt
 grep -q 'multiple: |one:two|' output.txt
 grep -q 'multiple_pos: |a:b:c:d:e:f|' output.txt
+grep -q 'meta_functionality_name: |testbash|' output.txt
+grep -q 'meta_resources_dir: |..*|' output.txt
 
 grep -q 'resources_dir: |..*|' output.txt
 grep -q 'head of input: |Scala|' output.txt
@@ -51,6 +53,9 @@ grep -q 'optional_with_default: |The default value.|' output2.txt
 grep -q 'multiple: ||' output2.txt
 grep -q 'multiple_pos: ||' output2.txt
 
+grep -q 'meta_functionality_name: |testbash|' output2.txt
+grep -q 'meta_resources_dir: |..*|' output2.txt
+
 grep -q 'resources_dir: |..*|' output2.txt
 grep -q 'head of input: |this file is only for testing|' output2.txt
 grep -q 'head of resource1: |if you can read this,|' output2.txt

diff --git a/src/test/resources/testcsharp/config.vsh.yaml b/src/test/resources/testcsharp/config.vsh.yaml
@@ -53,5 +53,8 @@ functionality:
 platforms:
   - type: native
   - type: docker
-    image: "andmos/dotnet-script"
+    image: "hjerpbakk/dotnet-script:1.0.1"
+    setup:
+      - type: apk
+        packages: [ bash ]
   - type: nextflow