diff --git a/.gitignore b/.gitignore index bb9d4171b..770bba5cc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,21 +1,29 @@ +# build related +bin/ +Makefile + +# sbt-related target/ -.bloop* -.metals* -projects/metals.sbt -/bin/ -.cache-main -.classpath -.project -.settings + +# Temporary files / folders output -.cache-tests -project/metals.sbt out/* -Makefile +# Rstudio-related .Rproj.user .Rhistory *.Rproj + +# Idea-related .idea -project/metals.sbt +.cache-main +.cache-tests +.classpath +.project +.settings + +# VS Code related .vscode/settings.json +**/metals.sbt +.bloop* +.metals* diff --git a/CHANGELOG.md b/CHANGELOG.md index f02ee5e3d..ee929fb18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,40 @@ +# Viash 0.5.10 + +## MAJOR CHANGES + +* `viash_install`: + - Added `--log_prefix`: This prefix is used to determine the path of the log files for `viash_build`, `viash_test` and `viash_push`. + - Added `--organization`: Id of the organisation to be used in the Docker image name, i.e. `//`. + - Added `--target_image_source`: Url to the Git repo in which this project resides. + - Removed `--log`. + +* `viash_build`: + - Reduce code duplication by contructing the command with Bash Arrays. + - Renamed `--platforms` to `--platform`. + - Added `--organization`: Id of the organisation to be used in the Docker image name, i.e. `//`. + - Added `--target_image_source`: Url to the Git repo in which this project resides. + - Changed default of `--log` from `log.txt` to `.viash_build_log.txt`. + - Added `--verbose`: Print out the underlying `viash ns build` command before running it. + +* `viash_test`: + - Reduce code duplication by contructing the command with Bash Arrays. + - Renamed `--platforms` to `--platform`. + - Added `--organization`: Id of the organisation to be used in the Docker image name, i.e. `//`. + - Added `--target_image_source`: Url to the Git repo in which this project resides. + - Changed default of `--log` from `log.txt` to `.viash_test_log.txt`. + - Changed default of `--tsv` from `log.tsv` to `.viash_test_log.tsv`. + - Added `--verbose`: Print out the underlying `viash ns test` command before running it. + +* `viash_push`: + - Reduce code duplication by contructing the command with Bash Arrays. + - Added `--organization`: Id of the organisation to be used in the Docker image name, i.e. `//`. + - Changed default of `--log` from `log.txt` to `.viash_push_log.txt`. + - Added `--verbose`: Print out the underlying `viash ns build` command before running it. + +## MINOR CHANGES + +* `NextflowPlatform`: Added the `organization` field to the nextflow platform as well. + # Viash 0.5.9 ## NEW FEATURES diff --git a/build.sbt b/build.sbt index f4da89acb..d99303dd8 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ name := "viash" -version := "0.5.9" +version := "0.5.10" scalaVersion := "2.12.10" diff --git a/src/main/scala/com/dataintuitive/viash/platforms/NextFlowPlatform.scala b/src/main/scala/com/dataintuitive/viash/platforms/NextFlowPlatform.scala index a1f18c58c..1098cb317 100644 --- a/src/main/scala/com/dataintuitive/viash/platforms/NextFlowPlatform.scala +++ b/src/main/scala/com/dataintuitive/viash/platforms/NextFlowPlatform.scala @@ -33,6 +33,7 @@ case class NextFlowPlatform( tag: Option[Version] = None, version: Option[Version] = None, registry: Option[String] = None, + organization: Option[String] = None, namespace_separator: String = "_", executor: Option[String] = None, publish: Option[Boolean] = None, @@ -63,6 +64,7 @@ case class NextFlowPlatform( val imageInfo = Docker.getImageInfo( functionality = Some(functionality), registry = registry, + organization = organization, name = image, tag = tag.map(_.toString), namespaceSeparator = namespace_separator diff --git a/src/viash/viash_build/config.vsh.yaml b/src/viash/viash_build/config.vsh.yaml index 4d2d4361c..b04f86f4f 100644 --- a/src/viash/viash_build/config.vsh.yaml +++ b/src/viash/viash_build/config.vsh.yaml @@ -1,7 +1,6 @@ functionality: name: viash_build namespace: viash - version: 0.1 description: | Build a project, usually in the context of a pipeline. arguments: @@ -15,39 +14,53 @@ functionality: type: string description: "The mode to run in. Possible values are: 'development', 'integration', 'release'." default: development - - name: "--platforms" + - name: "--platform" alternatives: [ "-p" ] type: string - description: "Which platforms to test, default is 'docker|nextflow'." - default: "docker|nextflow" + description: "Which platforms to process." + example: "docker|nextflow" - name: "--query" alternatives: [ "-q" ] type: string - description: "Filter which components get selected by name and namespace. Can be a regex. Example: '^mynamespace/component1$'." + description: "Filter which components get selected by name and namespace. Can be a regex." + example: "^mynamespace/component1$" - name: "--query_namespace" alternatives: [ "-n" ] type: string - description: "Filter which namespaces get selected by namespace. Can be a regex. Example: '^mynamespace$'." + description: "Filter which namespaces get selected by namespace. Can be a regex." + example: "^mynamespace$" - name: "--query_name" type: string - description: "Filter which components get selected by name. Can be a regex. Example: '^component1'." + description: "Filter which components get selected by name. Can be a regex." + example: "^component1$" - name: "--tag" alternatives: [ "-t" ] type: string description: The tag/version to be used. - default: "dev" + example: "0.1.0" - name: "--registry" alternatives: [ "-r" ] - default: "" + example: ghcr.io type: string - description: Docker registry to use, only used when using a registry. + description: Which Docker registry to use in the Docker image name. + - name: "--organization" + alternatives: [ "-o", "--organisation" ] + example: myorganisation + type: string + description: Which organisation name to use in the Docker image name. + - name: "--target_image_source" + alternatives: [ "-tis" ] + type: string + description: Which image source to specify in the component builds. + example: https://github.com/myorganisation/myrepository - name: "--namespace_separator" - default: "_" + example: "_" type: string description: The separator to use between the component name and namespace as the image name of a Docker container. - name: "--max_threads" type: integer - description: The maximum number of threads viash will use when `--parallell` during parallel tasks. + description: The maximum number of threads viash will use when `--parallel` during parallel tasks. + example: 8 - name: "--config_mod" alternatives: [ "-c" ] type: string @@ -62,11 +75,14 @@ functionality: alternatives: [ "-l" ] type: file description: Log file - default: log.txt + example: .viash_build_log.txt direction: output - name: "--viash" type: file description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. + - name: "--verbose" + type: boolean_true + description: "Increase verbosity." resources: - type: bash_script path: script.sh diff --git a/src/viash/viash_build/script.sh b/src/viash/viash_build/script.sh index 54c33b847..b60ceb48a 100644 --- a/src/viash/viash_build/script.sh +++ b/src/viash/viash_build/script.sh @@ -1,96 +1,141 @@ #!/bin/bash -# if not specified, default queries to a catch-all regexes -if [ -z "$par_query" ]; then - par_query=".*" +# start creating command +command_builder=( + ns build + --src "$par_src" + --parallel --write_meta +) + +# check par mode +if [ "$par_mode" == "development" ]; then + echo "In development mode with 'dev'." +elif [ "$par_mode" == "integration" ]; then + echo "In integration mode with tag '$par_tag'." +elif [ "$par_mode" == "release" ]; then + echo "In RELEASE mode with tag '$par_tag'." +else + echo "Error: Not a valid mode argument '$par_mode'." + exit 1 fi -if [ -z "$par_query_namespace" ]; then - par_query_namespace=".*" + +# check tag +if [ "$par_mode" == "development" ]; then + if [ ! -z "$par_tag" ]; then + echo "Warning: '--tag' is ignored when '--mode=$par_mode'." + fi + par_tag="dev" fi -if [ -z "$par_query_name" ]; then - par_query_name=".*" +if [ -z "$par_tag" ]; then + echo "Error: --tag is a requirement argument when '--mode=$par_mode'." + exit 1 fi +command_builder+=( + --config_mod ".functionality.version := '$par_tag'" +) + +# derive setup strategy +if [ "$par_mode" == "development" ]; then + if [ "$par_no_cache" == "true" ]; then + setup_strat="build" + else + setup_strat="cachedbuild" + fi +elif [ "$par_mode" == "integration" ]; then + echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." + setup_strat="ifneedbepullelsecachedbuild" +elif [ "$par_mode" == "release" ]; then + echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." + setup_strat="build" +fi + +command_builder+=( + --setup "$setup_strat" +) + +# check registry and organization +if [ "$par_mode" == "development" ]; then + if [ ! -z "$par_registry" ]; then + [[ "$par_verbose" == "true" ]] && echo "Note: --par_registry is ignored when '--mode=development'." + unset par_registry + fi + + if [ ! -z "$par_organization" ]; then + [[ "$par_verbose" == "true" ]] && echo "Note: --par_organization is ignored when '--mode=development'." + unset par_organization + fi +fi + +################ COMMON PARAMS ################ + +# check viash arg # if not specified, default par_viash to look for 'viash' on the PATH if [ -z "$par_viash" ]; then par_viash="viash" fi - # if specified, use par_max_threads as a java argument if [ ! -z "$par_max_threads" ]; then export JAVA_ARGS="$JAVA_ARGS -Dscala.concurrent.context.maxThreads=$par_max_threads" fi -if [ "$par_mode" == "release" ]; then - echo "In release mode with tag '$par_tag'." - if [ "$par_tag" == "dev" ]; then - echo "For a release, you have to specify an explicit version using --tag" - exit 1 - fi +# process queries +if [ ! -z "$par_query" ]; then + command_builder+=( "--query" "$par_query" ) +fi +if [ ! -z "$par_query_namespace" ]; then + command_builder+=( "--query_name" "$par_query_namespace" ) +fi +if [ ! -z "$par_query_name" ]; then + command_builder+=( "--query_namespace" "$par_query_name" ) fi -if [ "$par_mode" == "development" ]; then - echo "In development mode..." - - if [ "$par_no_cache" == "true" ]; then - setup_strat="build" - else - setup_strat="cachedbuild" - fi - - "$par_viash" ns build \ - -s "$par_src" \ - --platform "$par_platforms" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "dev"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l -w \ - --setup "$setup_strat" | tee "$par_log" -elif [ "$par_mode" == "integration" ]; then - echo "In integration mode..." - - if [ "$par_no_cache" == "true" ]; then - echo "Warning: '--no_cache' only applies when '--mode=development'." - fi - - "$par_viash" ns build \ - -s "$par_src" \ - --platform "$par_platforms" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "'"$par_tag"'"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "ifneedbepullelsecachedbuild"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l -w \ - --setup "build" | tee "$par_log" -elif [ "$par_mode" == "release" ]; then - - if [ "$par_no_cache" == "true" ]; then - echo "Warning: '--no_cache' only applies when '--mode=development'." - fi - - "$par_viash" ns build \ - -s "$par_src" \ - --platform "$par_platforms" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "'"$par_tag"'"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "ifneedbepullelsecachedbuild"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l -w \ - --setup "build" | tee "$par_log" -else - echo "Not a valid mode argument" +# process config mods +if [ ! -z "$par_config_mod" ]; then + IFS=";" + for var in $par_config_mod; do + unset IFS + command_builder+=( "--config_mod" "$var" ) + done +fi + +if [ ! -z "$par_registry" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_registry := '$par_registry'" + --config_mod ".platforms[.type == 'nextflow'].registry := '$par_registry'" + ) +fi + +if [ ! -z "$par_organization" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_organization := '$par_organization'" + --config_mod ".platforms[.type == 'nextflow'].organization := '$par_organization'" + ) +fi + +if [ ! -z "$par_namespace_separator" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker' || .type == 'nextflow'].namespace_separator := '$par_namespace_separator'" + ) fi + +if [ ! -z "$par_target_image_source" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_image_source := '$par_target_image_source'" + ) +fi + +if [ ! -z "$par_platform" ]; then + command_builder+=( --platform "$par_platform" ) +fi + +################ RUN COMMAND ################ +[[ "$par_verbose" == "true" ]] && echo "+ $par_viash" "${command_builder[@]}" + +if [ -z "$par_log" ]; then + "$par_viash" "${command_builder[@]}" +else + rm "$par_log" + "$par_viash" "${command_builder[@]}" > >(tee -a "$par_log") 2> >(tee -a "$par_log") +fi \ No newline at end of file diff --git a/src/viash/viash_clean_nxf/config.vsh.yaml b/src/viash/viash_clean_nxf/config.vsh.yaml index 1ff474744..a398882dd 100644 --- a/src/viash/viash_clean_nxf/config.vsh.yaml +++ b/src/viash/viash_clean_nxf/config.vsh.yaml @@ -1,7 +1,6 @@ functionality: name: viash_clean_nxf namespace: viash - version: 0.1 description: | Clean a (nextflow) project directory arguments: diff --git a/src/viash/viash_install/config.vsh.yaml b/src/viash/viash_install/config.vsh.yaml index 456982436..0ad958c7f 100644 --- a/src/viash/viash_install/config.vsh.yaml +++ b/src/viash/viash_install/config.vsh.yaml @@ -12,11 +12,21 @@ functionality: default: bin/ - name: "--registry" alternatives: [ "-r" ] - default: "" + example: ghcr.io type: string - description: Docker registry to use, only used when using a registry. + description: Which Docker registry to use in the Docker image name. + - name: "--organization" + alternatives: [ "-o", "--organisation" ] + example: myorganisation + type: string + description: Which organisation name to use in the Docker image name. + - name: "--target_image_source" + alternatives: [ "-tis" ] + type: string + description: Which image source to specify in the component builds. + example: https://github.com/myorganisation/myrepository - name: "--namespace_separator" - default: "_" + example: "_" type: string description: The separator to use between the component name and namespace as the image name of a Docker container. - name: "--config_mod" @@ -30,13 +40,11 @@ functionality: type: string description: "Which tag/version of viash to use." default: latest - - name: "--log" - type: file - description: Path to write the test logs to. - default: log.tsv - - name: "--viash" + - name: "--log_prefix" + alternatives: [ "-l" ] type: string - description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. + description: "A prefix path or directory for where to store the log files." + default: ./.viash_log_ resources: - type: bash_script path: script.sh diff --git a/src/viash/viash_install/script.sh b/src/viash/viash_install/script.sh index efe1d3d37..939858113 100644 --- a/src/viash/viash_install/script.sh +++ b/src/viash/viash_install/script.sh @@ -9,6 +9,9 @@ if ! command -v curl &> /dev/null; then exit fi +set -e + + # get the root of the repository REPO_ROOT=`pwd` @@ -57,7 +60,7 @@ if [ $par_tag == "develop" ]; then fi # Download Viash helper scripts - echo "> Downloading source v$par_tag" + echo "> Downloading Viash source code @$par_tag" curl -L -s "https://github.com/viash-io/viash/archive/refs/heads/$par_tag.zip" -o "$build_dir/$par_tag.zip" unzip -q "$build_dir/$par_tag.zip" -d "$build_dir" @@ -81,15 +84,37 @@ else fi # build components +extra_args=() + +if [ ! -z "$par_registry" ]; then + extra_args+=( -c ".functionality.arguments[.name == '--registry'].default := '$par_registry'" ) +fi +if [ ! -z "$par_organization" ]; then + extra_args+=( -c ".functionality.arguments[.name == '--organization'].default := '$par_organization'" ) +fi +if [ ! -z "$par_namespace_separator" ]; then + extra_args+=( -c ".functionality.arguments[.name == '--namespace_separator'].default := '$par_namespace_separator'" ) +fi +if [ ! -z "$par_config_mod" ]; then + echo "Warning: Adding config mods to viash install is currently not supported." + # extra_args+=( -c ".functionality.arguments[.name == '--config_mod'].default := '${par_config_mod@Q}'" ) +fi +if [ ! -z "$par_target_image_source" ]; then + extra_args+=( -c ".functionality.arguments[.name == '--target_image_source'].default := '$par_target_image_source'" ) +fi + + echo "> Building Viash helper scripts from source" "$par_bin/viash" ns build \ -s "$build_dir/viash-$par_tag/src/viash" \ -t "$par_bin/" \ --flatten \ - -c ".functionality.arguments[.name == '--registry'].default := '$par_registry'" \ - -c ".functionality.arguments[.name == '--viash'].default := '$par_viash'" \ - -c ".functionality.arguments[.name == '--log' && root.functionality.name == 'viash_test'].default := '$par_log'" \ - -c ".functionality.arguments[.name == '--namespace_separator'].default := '$par_namespace_separator'" \ + "${extra_args[@]}" \ + -c ".functionality.arguments[.name == '--viash'].default := '"$par_bin/viash"'" \ + -c ".functionality.arguments[.name == '--log' && root.functionality.name == 'viash_build'].default := '"${par_log_prefix}build.txt"'" \ + -c ".functionality.arguments[.name == '--log' && root.functionality.name == 'viash_test'].default := '"${par_log_prefix}test.txt"'" \ + -c ".functionality.arguments[.name == '--tsv' && root.functionality.name == 'viash_test'].default := '"${par_log_prefix}test.tsv"'" \ + -c ".functionality.arguments[.name == '--log' && root.functionality.name == 'viash_push'].default := '"${par_log_prefix}push.txt"'" \ -c ".functionality.version := '$par_tag'" echo "> Done, happy viash-ing!" diff --git a/src/viash/viash_push/config.vsh.yaml b/src/viash/viash_push/config.vsh.yaml index 0ca88ddbe..482dbfac2 100644 --- a/src/viash/viash_push/config.vsh.yaml +++ b/src/viash/viash_push/config.vsh.yaml @@ -18,33 +18,43 @@ functionality: - name: "--query" alternatives: [ "-q" ] type: string - description: "Filter which components get selected by name and namespace. Can be a regex. Example: '^mynamespace/component1$'." + description: "Filter which components get selected by name and namespace. Can be a regex." + example: "^mynamespace/component1$" - name: "--query_namespace" alternatives: [ "-n" ] type: string - description: "Filter which namespaces get selected by namespace. Can be a regex. Example: '^mynamespace$'." + description: "Filter which namespaces get selected by namespace. Can be a regex." + example: "^mynamespace$" - name: "--query_name" type: string - description: "Filter which components get selected by name. Can be a regex. Example: '^component1'." + description: "Filter which components get selected by name. Can be a regex." + example: "^component1$" - name: "--tag" alternatives: [ "-t" ] type: string description: The tag/version to be used. - default: "dev" + example: "dev" - name: "--registry" alternatives: [ "-r" ] + example: ghcr.io type: string - description: Docker registry to use, only used when using a registry. + description: Which Docker registry to use in the Docker image name. + - name: "--organization" + alternatives: [ "-o", "--organisation" ] + example: myorganisation + type: string + description: Which organisation name to use in the Docker image name. - name: "--namespace_separator" - default: "_" + example: "_" type: string description: The separator to use between the component name and namespace as the image name of a Docker container. - - name: "--force" - type: boolean_true - description: Overwrite registry - name: "--max_threads" type: integer - description: The maximum number of threads viash will use when `--parallell` during parallel tasks. + description: The maximum number of threads viash will use when `--parallel` during parallel tasks. + example: 8 + - name: "--force" + type: boolean_true + description: Overwrite existing container. - name: "--config_mod" alternatives: [ "-c" ] type: string @@ -55,10 +65,13 @@ functionality: alternatives: [ "-l" ] type: file description: Log file - default: log.txt + example: .viash_push_log.txt - name: "--viash" type: file description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. + - name: "--verbose" + type: boolean_true + description: "Increase verbosity." resources: - type: bash_script path: script.sh diff --git a/src/viash/viash_push/script.sh b/src/viash/viash_push/script.sh index b925af227..8d494dfd9 100644 --- a/src/viash/viash_push/script.sh +++ b/src/viash/viash_push/script.sh @@ -1,108 +1,119 @@ #!/bin/bash -if [ "$par_mode" == "release" ]; then - echo "In release mode with tag '$par_tag'." - if [ "$par_tag" == "dev" ]; then - echo "For a release, you have to specify an explicit version using --tag" - exit 1 - fi +if [ "$par_force" == "true" ]; then + echo "Force push... handle with care..." + sleep 2 fi -# if not specified, default queries to a catch-all regexes -if [ -z "$par_query" ]; then - par_query=".*" +# start creating command +command_builder=( + ns build + --src "$par_src" + --platform docker + --parallel +) + +# check par mode +if [ "$par_mode" == "development" ]; then + echo "No container push can and should be performed in this mode." + exit 1 +elif [ "$par_mode" == "integration" ]; then + echo "No container push can and should be performed in this mode." + exit 1 + # if [ ! -z "$par_tag" ]; then + # echo "Warning: '--tag' is ignored when '--mode=$par_mode'." + # fi + # par_tag="dev" + # echo "In integration mode with tag '$par_tag'." +elif [ "$par_mode" == "release" ]; then + echo "In RELEASE mode with tag '$par_tag'." +else + echo "Error: Not a valid mode argument '$par_mode'." + exit 1 fi -if [ -z "$par_query_namespace" ]; then - par_query_namespace=".*" + +if [ -z "$par_tag" ]; then + echo "Error: --tag is a requirement argument when '--mode=$par_mode'." + exit 1 fi -if [ -z "$par_query_name" ]; then - par_query_name=".*" + +command_builder+=( + --config_mod ".functionality.version := '$par_tag'" +) + +# derive setup strategy +if [ "$par_force" == "true" ]; then + setup_strat="push" +else + setup_strat="pushifnotpresent" fi +command_builder+=( + --setup "$setup_strat" +) + + +################ COMMON PARAMS ################ + +# check viash arg # if not specified, default par_viash to look for 'viash' on the PATH if [ -z "$par_viash" ]; then par_viash="viash" fi - # if specified, use par_max_threads as a java argument if [ ! -z "$par_max_threads" ]; then export JAVA_ARGS="$JAVA_ARGS -Dscala.concurrent.context.maxThreads=$par_max_threads" fi -if [[ $par_force == true ]]; then - echo "Force push... handle with care..." - if [ "$par_mode" == "development" ]; then - echo "No container push can and should be performed in this mode" - elif [ "$par_mode" == "integration" ]; then - "$par_viash" ns build \ - -s "$par_src" \ - --platform "docker" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "dev"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "donothing"' \ - -c '.platforms[.type == "docker"].push_strategy := "alwayspush"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - --setup "push" | tee "$par_log" - elif [ "$par_mode" == "release" ]; then - "$par_viash" ns build \ - -s "$par_src" \ - --platform "docker" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "'"$par_tag"'"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "donothing"' \ - -c '.platforms[.type == "docker"].push_strategy := "alwayspush"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - --setup "push" | tee "$par_log" - else - echo "Not a valid mode argument" - fi -else - if [ "$par_mode" == "development" ]; then - echo "No container push can and should be performed in this mode" - elif [ "$par_mode" == "integration" ]; then - "$par_viash" ns build \ - -s "$par_src" \ - --platform "docker" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "dev"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "donothing"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - --setup "push" | tee "$par_log" - elif [ "$par_mode" == "release" ]; then - "$par_viash" ns build \ - -s "$par_src" \ - --platform "docker" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "'"$par_tag"'"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "donothing"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - --setup "push" | tee "$par_log" - else - echo "Not a valid mode argument" - fi +# process queries +if [ ! -z "$par_query" ]; then + command_builder+=( "--query" "$par_query" ) +fi +if [ ! -z "$par_query_namespace" ]; then + command_builder+=( "--query_name" "$par_query_namespace" ) fi +if [ ! -z "$par_query_name" ]; then + command_builder+=( "--query_namespace" "$par_query_name" ) +fi + +# process config mods +if [ ! -z "$par_config_mod" ]; then + IFS=";" + for var in $par_config_mod; do + unset IFS + command_builder+=( "--config_mod" "$var" ) + done +fi + +if [ ! -z "$par_registry" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_registry := '$par_registry'" + --config_mod ".platforms[.type == 'nextflow'].registry := '$par_registry'" + ) +fi + +if [ ! -z "$par_organization" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_organization := '$par_organization'" + --config_mod ".platforms[.type == 'nextflow'].organization := '$par_organization'" + ) +fi + +if [ ! -z "$par_namespace_separator" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker' || .type == 'nextflow'].namespace_separator := '$par_namespace_separator'" + ) +fi + + + +################ RUN COMMAND ################ +[[ "$par_verbose" == "true" ]] && echo "+ $par_viash" "${command_builder[@]}" + +if [ -z "$par_log" ]; then + "$par_viash" "${command_builder[@]}" +else + rm "$par_log" + "$par_viash" "${command_builder[@]}" > >(tee -a "$par_log") 2> >(tee -a "$par_log") +fi \ No newline at end of file diff --git a/src/viash/viash_skeleton/script.sh b/src/viash/viash_skeleton/script.sh index bbe2cc8ca..20cde3d46 100644 --- a/src/viash/viash_skeleton/script.sh +++ b/src/viash/viash_skeleton/script.sh @@ -45,12 +45,14 @@ cat >> "$out_dir/config.vsh.yaml" << HERE type: file required: true description: Describe the input file. + example: input.txt - name: "--output" alternatives: [ "-o" ] type: file direction: output required: true description: Describe the output file. + example: output.txt - name: "--option" type: string description: Describe an optional parameter. diff --git a/src/viash/viash_test/config.vsh.yaml b/src/viash/viash_test/config.vsh.yaml index d13e9d94f..a744a6604 100644 --- a/src/viash/viash_test/config.vsh.yaml +++ b/src/viash/viash_test/config.vsh.yaml @@ -15,49 +15,68 @@ functionality: type: string description: "The mode to run in. Possible values are: 'development', 'integration', 'release'." default: development - - name: "--platforms" + - name: "--platform" alternatives: [ "-p" ] type: string - description: Which platforms to test, default is 'docker'. - default: "docker" + description: "Which platforms to process." + default: "docker" # to do: should be set to 'example' - name: "--query" alternatives: [ "-q" ] type: string - description: "Filter which components get selected by name and namespace. Can be a regex. Example: '^mynamespace/component1$'." + description: "Filter which components get selected by name and namespace. Can be a regex." + example: "^mynamespace/component1$" - name: "--query_namespace" alternatives: [ "-n" ] type: string - description: "Filter which namespaces get selected by namespace. Can be a regex. Example: '^mynamespace$'." + description: "Filter which namespaces get selected by namespace. Can be a regex." + example: "^mynamespace$" - name: "--query_name" type: string - description: "Filter which components get selected by name. Can be a regex. Example: '^component1'." + description: "Filter which components get selected by name. Can be a regex." + example: "^component1$" - name: "--tag" alternatives: [ "-t" ] type: string description: Which tag/version of the pipeline to use. - default: "dev" + example: "0.1.0" - name: "--registry" alternatives: [ "-r" ] + example: ghcr.io type: string - description: Docker registry to use, only used when using a registry. + description: Which Docker registry to use in the Docker image name. + - name: "--organization" + alternatives: [ "-o", "--organisation" ] + example: myorganisation + type: string + description: Which organisation name to use in the Docker image name. + - name: "--target_image_source" + alternatives: [ "-tis" ] + type: string + description: Which image source to specify in the component builds. + example: https://github.com/myorganisation/myrepository - name: "--namespace_separator" - default: "_" + example: "_" type: string description: The separator to use between the component name and namespace as the image name of a Docker container. - name: "--max_threads" type: integer - description: The maximum number of threads viash will use when `--parallell` during parallel tasks. + description: The maximum number of threads viash will use when `--parallel` during parallel tasks. + example: 8 - name: "--config_mod" alternatives: [ "-c" ] type: string multiple: true multiple_sep: ";" description: "Modify a viash config at runtime using a custom DSL. For more information, see the online documentation." + - name: "--tsv" + type: file + description: Test results stored as a tabular text file. + example: .viash_test_log.tsv - name: "--log" alternatives: [ "-l" ] type: file description: Test log file - default: log.tsv + example: .viash_test_log.txt - name: "--append" type: boolean default: true @@ -65,6 +84,9 @@ functionality: - name: "--viash" type: file description: A path to the viash executable. If not specified, this component will look for 'viash' on the $PATH. + - name: "--verbose" + type: boolean_true + description: "Increase verbosity." resources: - type: bash_script path: script.sh diff --git a/src/viash/viash_test/script.sh b/src/viash/viash_test/script.sh index d56e025c3..ab953a04b 100644 --- a/src/viash/viash_test/script.sh +++ b/src/viash/viash_test/script.sh @@ -1,88 +1,146 @@ #!/bin/bash -# if not specified, default queries to a catch-all regexes -if [ -z "$par_query" ]; then - par_query=".*" +# start creating command +command_builder=( + ns test + --src "$par_src" + --parallel +) + +# check par mode +if [ "$par_mode" == "development" ]; then + echo "In development mode with 'dev'." +elif [ "$par_mode" == "integration" ]; then + echo "In integration mode with tag '$par_tag'." +elif [ "$par_mode" == "release" ]; then + echo "In RELEASE mode with tag '$par_tag'." +else + echo "Error: Not a valid mode argument '$par_mode'." + exit 1 fi -if [ -z "$par_query_namespace" ]; then - par_query_namespace=".*" + +# check tag +if [ "$par_mode" == "development" ]; then + if [ ! -z "$par_tag" ]; then + echo "Warning: '--tag' is ignored when '--mode=$par_mode'." + fi + par_tag="dev" fi -if [ -z "$par_query_name" ]; then - par_query_name=".*" +if [ -z "$par_tag" ]; then + echo "Error: --tag is a requirement argument when '--mode=$par_mode'." + exit 1 fi -# if not specified, default par_viash to look for 'viash' on the PATH -if [ -z "$par_viash" ]; then - par_viash="viash" +# derive setup strategy +if [ "$par_mode" == "development" ]; then + if [ "$par_no_cache" == "true" ]; then + setup_strat="build" + else + setup_strat="cachedbuild" + fi +elif [ "$par_mode" == "integration" ]; then + echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." + setup_strat="ifneedbepullelsecachedbuild" +elif [ "$par_mode" == "release" ]; then + echo "Warning: --par_no_cache is ignored when '--mode=$par_mode'." + setup_strat="build" fi -# if --append (-a) true is specified, add `--append` -if [ "$par_append" == "true" ]; then - par_append_parsed="--append" +command_builder+=( + --config_mod ".functionality.version := '$par_tag'" + --config_mod ".platforms[.type == 'docker'].setup_strategy := '$setup_strat'" +) + +# check registry and organization +if [ "$par_mode" == "development" ]; then + if [ ! -z "$par_registry" ]; then + [[ "$par_verbose" == "true" ]] && echo "Note: --par_registry is ignored when '--mode=development'." + unset par_registry + fi + + if [ ! -z "$par_organization" ]; then + [[ "$par_verbose" == "true" ]] && echo "Note: --par_organization is ignored when '--mode=development'." + unset par_organization + fi fi +################ COMMON PARAMS ################ + +# check viash arg +# if not specified, default par_viash to look for 'viash' on the PATH +if [ -z "$par_viash" ]; then + par_viash="viash" +fi # if specified, use par_max_threads as a java argument if [ ! -z "$par_max_threads" ]; then export JAVA_ARGS="$JAVA_ARGS -Dscala.concurrent.context.maxThreads=$par_max_threads" fi -if [ "$par_mode" == "release" ]; then - echo "In release mode with tag '$par_tag'." - if [ "$par_tag" == "dev" ]; then - echo "For a release, you have to specify an explicit version using --tag" - exit 1 - fi +# process queries +if [ ! -z "$par_query" ]; then + command_builder+=( "--query" "$par_query" ) +fi +if [ ! -z "$par_query_namespace" ]; then + command_builder+=( "--query_name" "$par_query_namespace" ) +fi +if [ ! -z "$par_query_name" ]; then + command_builder+=( "--query_namespace" "$par_query_name" ) fi -if [ "$par_mode" == "development" ]; then - echo "In development mode..." - "$par_viash" ns test \ - -s "$par_src" \ - --platform "$par_platforms" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "dev"' \ - -c '.platforms[.type == "docker"].setup_strategy := "donothing"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - -t "$par_log" \ - $par_append_parsed -elif [ "$par_mode" == "integration" ]; then - echo "In integration mode..." - "$par_viash" ns test \ - -s "$par_src" \ - --platform "$par_platforms" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "'"$par_tag"'"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "donothing"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - -t "$par_log" \ - $par_append_parsed -elif [ "$par_mode" == "release" ]; then - "$par_viash" ns test \ - -s "$par_src" \ - --platform "$par_platforms" \ - --query "$par_query" \ - --query_name "$par_query_name" \ - --query_namespace "$par_query_namespace" \ - -c '.functionality.version := "'"$par_tag"'"' \ - -c '.platforms[.type == "docker"].target_registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker"].setup_strategy := "pull"' \ - -c '.platforms[.type == "nextflow"].registry := "'"$par_registry"'"' \ - -c '.platforms[.type == "docker" || .type == "nextflow"].namespace_separator := "'$par_namespace_separator'"' \ - -c "$par_config_mod" \ - -l \ - -t "$par_log" \ - $par_append_parsed -else - echo "Not a valid mode argument" +# process config mods +if [ ! -z "$par_config_mod" ]; then + IFS=";" + for var in $par_config_mod; do + unset IFS + command_builder+=( "--config_mod" "$var" ) + done +fi + +if [ ! -z "$par_registry" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_registry := '$par_registry'" + --config_mod ".platforms[.type == 'nextflow'].registry := '$par_registry'" + ) +fi + +if [ ! -z "$par_organization" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_organization := '$par_organization'" + --config_mod ".platforms[.type == 'nextflow'].organization := '$par_organization'" + ) +fi + +if [ ! -z "$par_namespace_separator" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker' || .type == 'nextflow'].namespace_separator := '$par_namespace_separator'" + ) +fi + +if [ ! -z "$par_target_image_source" ]; then + command_builder+=( + --config_mod ".platforms[.type == 'docker'].target_image_source := '$par_target_image_source'" + ) +fi + +if [ ! -z "$par_platform" ]; then + command_builder+=( --platform "$par_platform" ) +fi + +if [ "$par_organization" == "true" ]; then + command_builder+=("--append") +fi + +if [ ! -z "$par_tsv" ]; then + command_builder+=( --tsv "$par_tsv" ) fi + +################ RUN COMMAND ################ +[[ "$par_verbose" == "true" ]] && echo "+ $par_viash" "${command_builder[@]}" + +if [ -z "$par_log" ]; then + "$par_viash" "${command_builder[@]}" +else + rm "$par_log" + "$par_viash" "${command_builder[@]}" > >(tee -a "$par_log") 2> >(tee -a "$par_log") +fi \ No newline at end of file