diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 2afdf3d22..29f281516 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -173,6 +173,8 @@ jobs: - cwl-version: v1.2 container: docker extras: "--fast-parser" + - cwl-version: v1.3.0-dev1 + extras: "--relax-path-checks" steps: - uses: actions/checkout@v4 diff --git a/MANIFEST.in b/MANIFEST.in index 92f65cfe5..187d19bea 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ recursive-include mypy-stubs *.pyi *.py include tests/* include tests/cwl-conformance/cwltool-conftest.py include tests/loop/* +include tests/loop-ext/* include tests/tmp1/tmp2/tmp3/.gitkeep include tests/tmp4/alpha/* include tests/wf/* @@ -54,6 +55,10 @@ include cwltool/schemas/v1.2/*.yml include cwltool/schemas/v1.2/*.md include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.yml include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.md +include cwltool/schemas/v1.3.0-dev1/*.yml +include cwltool/schemas/v1.3.0-dev1/*.md +include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.yml +include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.md include cwltool/extensions.yml include cwltool/extensions-v1.1.yml include cwltool/extensions-v1.2.yml diff --git a/Makefile b/Makefile index 92e51f493..1b08f4290 100644 --- a/Makefile +++ b/Makefile @@ -160,11 +160,11 @@ diff-cover.html: coverage.xml ## test : run the cwltool test suite test: $(PYSOURCES) - python3 -m pytest -rs ${PYTEST_EXTRA} + python3 -m pytest -rsfE ${PYTEST_EXTRA} ## testcov : run the cwltool test suite and collect coverage testcov: $(PYSOURCES) - python3 -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} + python3 -m pytest -rsfE --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA} sloccount.sc: $(PYSOURCES) Makefile sloccount --duplicates --wide --details $^ > $@ @@ -183,7 +183,7 @@ mypy: $(PYSOURCES) mypyc: $(PYSOURCES) MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \ - && pytest -rs -vv ${PYTEST_EXTRA} + && pytest -rsfE -vv ${PYTEST_EXTRA} shellcheck: FORCE shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \ diff --git a/conformance-test.sh b/conformance-test.sh index cdc57f69f..36ea23b17 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -16,7 +16,7 @@ venv() { # VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh # Version of the standard to test against -# Current options: v1.0, v1.1, v1.2 +# Current options: v1.0, v1.1, v1.2, v1.3.0-dev1 VERSION=${VERSION:-"v1.2"} # Which commit of the standard's repo to use @@ -78,7 +78,7 @@ else pip uninstall -y cwltool pip install -r"${SCRIPT_DIRECTORY}/mypy-requirements.txt" CWLTOOL_USE_MYPYC=1 MYPYPATH="${SCRIPT_DIRECTORY}/mypy-stubs" pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt" - pip install 'cwltest>=2.5' pytest-cov pytest-xdist>=3.2.0 psutil + pip install 'cwltest>=2.5,!=2.5.20240709060407' pytest-cov pytest-xdist>=3.2.0 psutil fi # Set conformance test filename @@ -121,7 +121,7 @@ if (( "${#exclusions[*]}" > 0 )); then fi # Build command -TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)" +TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rsfE --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)" if [[ -n "${EXCLUDE}" ]] ; then TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}" fi diff --git a/cwltool/checker.py b/cwltool/checker.py index c386409af..541d59440 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -517,42 +517,33 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) - def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: - """Check if a step contains a http://commonwl.org/cwltool#Loop requirement with `all` outputMethod.""" + """Check if a step contains a `loop` directive with `all` outputMethod.""" source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) if source_step is not None: - for requirement in source_step.get("requirements", []): - if ( - requirement["class"] == "http://commonwl.org/cwltool#Loop" - and requirement.get("outputMethod") == "all" - ): - return True + if source_step.get("loop") is not None and source_step.get("outputMethod") == "all": + return True return False def loop_checker(steps: Iterator[MutableMapping[str, Any]]) -> None: """ - Check http://commonwl.org/cwltool#Loop requirement compatibility with other directives. + Check `loop` compatibility with other directives. - :raises ValidationException: If there is an incompatible combination between - cwltool:loop and 'scatter' or 'when'. + :raises ValidationException: If there is an incompatible combination between `loop` and `scatter`. """ exceptions = [] for step in steps: - requirements = { - **{h["class"]: h for h in step.get("hints", [])}, - **{r["class"]: r for r in step.get("requirements", [])}, - } - if "http://commonwl.org/cwltool#Loop" in requirements: - if "when" in step: + if "loop" in step: + if "when" not in step: exceptions.append( SourceLine(step, "id").makeError( - "The `cwltool:Loop` clause is not compatible with the `when` directive." + "The `when` clause is mandatory when the `loop` directive is defined." ) ) if "scatter" in step: exceptions.append( SourceLine(step, "id").makeError( - "The `cwltool:Loop` clause is not compatible with the `scatter` directive." + "The `loop` clause is not compatible with the `scatter` directive." ) ) if exceptions: diff --git a/cwltool/errors.py b/cwltool/errors.py index a39fb3bc9..978078ec6 100644 --- a/cwltool/errors.py +++ b/cwltool/errors.py @@ -1,5 +1,17 @@ -class WorkflowException(Exception): - pass +"""Common errors. + +WorkflowException and GraphTargetMissingException are aliased to +equivalent errors from cwl_utils.errors and re-exported by this module +to avoid breaking the interface for other code. + +""" + +# flake8: noqa: F401 + +from cwl_utils.errors import WorkflowException as WorkflowException + + +from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException class UnsupportedRequirement(WorkflowException): @@ -8,7 +20,3 @@ class UnsupportedRequirement(WorkflowException): class ArgumentException(Exception): """Mismatched command line arguments provided.""" - - -class GraphTargetMissingException(WorkflowException): - """When a ``$graph`` is encountered and there is no target and no ``main``/``#main``.""" diff --git a/cwltool/extensions-v1.3.yml b/cwltool/extensions-v1.3.yml new file mode 100644 index 000000000..603c40f05 --- /dev/null +++ b/cwltool/extensions-v1.3.yml @@ -0,0 +1,121 @@ +$base: http://commonwl.org/cwltool# +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" +$graph: +- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml + +- name: Secrets + type: record + inVocab: false + extends: cwl:ProcessRequirement + fields: + class: + type: string + doc: "Always 'Secrets'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + secrets: + type: string[] + doc: | + List one or more input parameters that are sensitive (such as passwords) + which will be deliberately obscured from logging. + jsonldPredicate: + "_type": "@id" + refScope: 0 + + +- name: ProcessGenerator + type: record + inVocab: true + extends: cwl:Process + documentRoot: true + fields: + - name: class + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + type: string + - name: run + type: [string, cwl:Process] + jsonldPredicate: + _id: "cwl:run" + _type: "@id" + subscope: run + doc: | + Specifies the process to run. + +- name: MPIRequirement + type: record + inVocab: false + extends: cwl:ProcessRequirement + doc: | + Indicates that a process requires an MPI runtime. + fields: + - name: class + type: string + doc: "Always 'MPIRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: processes + type: [int, cwl:Expression] + doc: | + The number of MPI processes to start. If you give a string, + this will be evaluated as a CWL Expression and it must + evaluate to an integer. + +- name: CUDARequirement + type: record + extends: cwl:ProcessRequirement + inVocab: false + doc: | + Require support for NVIDA CUDA (GPU hardware acceleration). + fields: + class: + type: string + doc: 'cwltool:CUDARequirement' + jsonldPredicate: + _id: "@type" + _type: "@vocab" + cudaVersionMin: + type: string + doc: | + Minimum CUDA version to run the software, in X.Y format. This + corresponds to a CUDA SDK release. When running directly on + the host (not in a container) the host must have a compatible + CUDA SDK (matching the exact version, or, starting with CUDA + 11.3, matching major version). When run in a container, the + container image should provide the CUDA runtime, and the host + driver is injected into the container. In this case, because + CUDA drivers are backwards compatible, it is possible to + use an older SDK with a newer driver across major versions. + + See https://docs.nvidia.com/deploy/cuda-compatibility/ for + details. + cudaComputeCapability: + type: + - 'string' + - 'string[]' + doc: | + CUDA hardware capability required to run the software, in X.Y + format. + + * If this is a single value, it defines only the minimum + compute capability. GPUs with higher capability are also + accepted. + + * If it is an array value, then only select GPUs with compute + capabilities that explicitly appear in the array. + cudaDeviceCountMin: + type: ['null', int, cwl:Expression] + default: 1 + doc: | + Minimum number of GPU devices to request. If not specified, + same as `cudaDeviceCountMax`. If neither are specified, + default 1. + cudaDeviceCountMax: + type: ['null', int, cwl:Expression] + doc: | + Maximum number of GPU devices to request. If not specified, + same as `cudaDeviceCountMin`. diff --git a/cwltool/job.py b/cwltool/job.py index dd10dda1d..8a0d83090 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -917,7 +917,7 @@ def docker_monitor( self.name, int((max_mem_percent / 100 * max_mem) / (2**20)), ) - if cleanup_cidfile: + if cleanup_cidfile and os.path.exists(cidfile): os.remove(cidfile) diff --git a/cwltool/process.py b/cwltool/process.py index 42d90a395..afbed8147 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -125,6 +125,7 @@ def filter(self, record: logging.LogRecord) -> bool: ] cwl_files = ( + "Base.yml", "Workflow.yml", "CommandLineTool.yml", "CommonWorkflowLanguage.yml", @@ -1046,10 +1047,6 @@ def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict: sl = SourceLine(hints, i, ValidationException, debug) with sl: classname = cast(str, r["class"]) - if classname == "http://commonwl.org/cwltool#Loop": - raise ValidationException( - "http://commonwl.org/cwltool#Loop is valid only under requirements." - ) avroname = classname if classname in self.doc_loader.vocab: avroname = avro_type_name(self.doc_loader.vocab[classname]) diff --git a/cwltool/schemas/v1.3.0-dev1/Base.yml b/cwltool/schemas/v1.3.0-dev1/Base.yml new file mode 100644 index 000000000..c2ca0d5b9 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/Base.yml @@ -0,0 +1,498 @@ +$base: "https://w3id.org/cwl/cwl#" + +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + sld: "https://w3id.org/cwl/salad#" + +$graph: + +- name: CWLType + type: enum + extends: "sld:PrimitiveType" + symbols: + - cwl:File + - cwl:Directory + doc: + - "Extends primitive types with the concept of a file and directory as a builtin type." + - "File: A File object" + - "Directory: A Directory object" + +- name: CWLArraySchema + type: record + extends: "sld:ArraySchema" + fields: + items: + type: + - PrimitiveType + - CWLRecordSchema + - EnumSchema + - CWLArraySchema + - string + - type: array + items: + - PrimitiveType + - CWLRecordSchema + - EnumSchema + - CWLArraySchema + - string + jsonldPredicate: + _id: "sld:items" + _type: "@vocab" + refScope: 2 + doc: "Defines the type of the array elements." + +- name: CWLRecordField + type: record + extends: "sld:RecordField" + fields: + - name: type + type: + - PrimitiveType + - CWLRecordSchema + - EnumSchema + - CWLArraySchema + - string + - type: array + items: + - PrimitiveType + - CWLRecordSchema + - EnumSchema + - CWLArraySchema + - string + jsonldPredicate: + _id: sld:type + _type: "@vocab" + typeDSL: true + refScope: 2 + doc: | + The field type + +- name: CWLRecordSchema + type: record + extends: "sld:RecordSchema" + fields: + fields: + type: CWLRecordField[]? + jsonldPredicate: + _id: sld:fields + mapSubject: name + mapPredicate: type + doc: "Defines the fields of the record." + +- name: File + type: record + docParent: "#CWLType" + doc: | + Represents a file (or group of files when `secondaryFiles` is provided) that + will be accessible by tools using standard POSIX file system call API such as + open(2) and read(2). + + Files are represented as objects with `class` of `File`. File objects have + a number of properties that provide metadata about the file. + + The `location` property of a File is a IRI that uniquely identifies the + file. Implementations must support the `file://` IRI scheme and may support + other schemes such as `http://` and `https://`. The value of `location` may also be a + relative reference, in which case it must be resolved relative to the IRI + of the document it appears in. Alternately to `location`, implementations + must also accept the `path` property on File, which must be a filesystem + path available on the same host as the CWL runner (for inputs) or the + runtime environment of a command line tool execution (for command line tool + outputs). + + If no `location` or `path` is specified, a file object must specify + `contents` with the UTF-8 text content of the file. This is a "file + literal". File literals do not correspond to external resources, but are + created on disk with `contents` with when needed for executing a tool. + Where appropriate, expressions can return file literals to define new files + on a runtime. The maximum size of `contents` is 64 kilobytes. + + The `basename` property defines the filename on disk where the file is + staged. This may differ from the resource name. If not provided, + `basename` must be computed from the last path part of `location` and made + available to expressions. + + The `secondaryFiles` property is a list of File or Directory objects that + must be staged in the same directory as the primary file. It is an error + for file names to be duplicated in `secondaryFiles`. + + The `size` property is the size in bytes of the File. It must be computed + from the resource and made available to expressions. The `checksum` field + contains a cryptographic hash of the file content for use it verifying file + contents. Implementations may, at user option, enable or disable + computation of the `checksum` field for performance or other reasons. + However, the ability to compute output checksums is required to pass the + CWL conformance test suite. + + When executing a CommandLineTool, the files and secondary files may be + staged to an arbitrary directory, but must use the value of `basename` for + the filename. The `path` property must be file path in the context of the + tool execution runtime (local to the compute node, or within the executing + container). All computed properties should be available to expressions. + File literals also must be staged and `path` must be set. + + When collecting CommandLineTool outputs, `glob` matching returns file paths + (with the `path` property) and the derived properties. This can all be + modified by `outputEval`. Alternately, if the file `cwl.output.json` is + present in the output, `outputBinding` is ignored. + + File objects in the output must provide either a `location` IRI or a `path` + property in the context of the tool execution runtime (local to the compute + node, or within the executing container). + + When evaluating an ExpressionTool, file objects must be referenced via + `location` (the expression tool does not have access to files on disk so + `path` is meaningless) or as file literals. It is legal to return a file + object with an existing `location` but a different `basename`. The + `loadContents` field of ExpressionTool inputs behaves the same as on + CommandLineTool inputs, however it is not meaningful on the outputs. + + An ExpressionTool may forward file references from input to output by using + the same value for `location`. + + fields: + - name: class + type: + type: enum + name: File_class + symbols: + - cwl:File + jsonldPredicate: + _id: "@type" + _type: "@vocab" + doc: Must be `File` to indicate this object describes a file. + - name: location + type: string? + doc: | + An IRI that identifies the file resource. This may be a relative + reference, in which case it must be resolved using the base IRI of the + document. The location may refer to a local or remote resource; the + implementation must use the IRI to retrieve file content. If an + implementation is unable to retrieve the file content stored at a + remote resource (due to unsupported protocol, access denied, or other + issue) it must signal an error. + + If the `location` field is not provided, the `contents` field must be + provided. The implementation must assign a unique identifier for + the `location` field. + + If the `path` field is provided but the `location` field is not, an + implementation may assign the value of the `path` field to `location`, + then follow the rules above. + jsonldPredicate: + _id: "@id" + _type: "@id" + - name: path + type: string? + doc: | + The local host path where the File is available when a CommandLineTool is + executed. This field must be set by the implementation. The final + path component must match the value of `basename`. This field + must not be used in any other context. The command line tool being + executed must be able to access the file at `path` using the POSIX + `open(2)` syscall. + + As a special case, if the `path` field is provided but the `location` + field is not, an implementation may assign the value of the `path` + field to `location`, and remove the `path` field. + + If the `path` contains [POSIX shell metacharacters](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02) + (`|`,`&`, `;`, `<`, `>`, `(`,`)`, `$`,`` ` ``, `\`, `"`, `'`, + ``, ``, and ``) or characters + [not allowed](http://www.iana.org/assignments/idna-tables-6.3.0/idna-tables-6.3.0.xhtml) + for [Internationalized Domain Names for Applications](https://tools.ietf.org/html/rfc6452) + then implementations may terminate the process with a + `permanentFailure`. + jsonldPredicate: + "_id": "cwl:path" + "_type": "@id" + - name: basename + type: string? + doc: | + The base name of the file, that is, the name of the file without any + leading directory path. The base name must not contain a slash `/`. + + If not provided, the implementation must set this field based on the + `location` field by taking the final path component after parsing + `location` as an IRI. If `basename` is provided, it is not required to + match the value from `location`. + + When this file is made available to a CommandLineTool, it must be named + with `basename`, i.e. the final component of the `path` field must match + `basename`. + jsonldPredicate: "cwl:basename" + - name: dirname + type: string? + doc: | + The name of the directory containing file, that is, the path leading up + to the final slash in the path such that `dirname + '/' + basename == + path`. + + The implementation must set this field based on the value of `path` + prior to evaluating parameter references or expressions in a + CommandLineTool document. This field must not be used in any other + context. + - name: nameroot + type: string? + doc: | + The basename root such that `nameroot + nameext == basename`, and + `nameext` is empty or begins with a period and contains at most one + period. For the purposes of path splitting leading periods on the + basename are ignored; a basename of `.cshrc` will have a nameroot of + `.cshrc`. + + The implementation must set this field automatically based on the value + of `basename` prior to evaluating parameter references or expressions. + - name: nameext + type: string? + doc: | + The basename extension such that `nameroot + nameext == basename`, and + `nameext` is empty or begins with a period and contains at most one + period. Leading periods on the basename are ignored; a basename of + `.cshrc` will have an empty `nameext`. + + The implementation must set this field automatically based on the value + of `basename` prior to evaluating parameter references or expressions. + - name: checksum + type: string? + doc: | + Optional hash code for validating file integrity. Currently, must be in the form + "sha1$ + hexadecimal string" using the SHA-1 algorithm. + - name: size + type: + - "null" + - int + - long + doc: Optional file size (in bytes) + - name: "secondaryFiles" + type: + - "null" + - type: array + items: [File, Directory] + jsonldPredicate: + _id: "cwl:secondaryFiles" + secondaryFilesDSL: true + doc: | + A list of additional files or directories that are associated with the + primary file and must be transferred alongside the primary file. + Examples include indexes of the primary file, or external references + which must be included when loading primary document. A file object + listed in `secondaryFiles` may itself include `secondaryFiles` for + which the same rules apply. + - name: format + type: string? + jsonldPredicate: + _id: cwl:format + _type: "@id" + identity: true + noLinkCheck: true + doc: | + The format of the file: this must be an IRI of a concept node that + represents the file format, preferably defined within an ontology. + If no ontology is available, file formats may be tested by exact match. + + Reasoning about format compatibility must be done by checking that an + input file format is the same, `owl:equivalentClass` or + `rdfs:subClassOf` the format required by the input parameter. + `owl:equivalentClass` is transitive with `rdfs:subClassOf`, e.g. if + ` owl:equivalentClass ` and ` owl:subclassOf ` then infer + ` owl:subclassOf `. + + File format ontologies may be provided in the "$schemas" metadata at the + root of the document. If no ontologies are specified in `$schemas`, the + runtime may perform exact file format matches. + - name: contents + type: string? + doc: | + File contents literal. + + If neither `location` nor `path` is provided, `contents` must be + non-null. The implementation must assign a unique identifier for the + `location` field. When the file is staged as input to CommandLineTool, + the value of `contents` must be written to a file. + + If `contents` is set as a result of a Javascript expression, + an `entry` in `InitialWorkDirRequirement`, or read in from + `cwl.output.json`, there is no specified upper limit on the + size of `contents`. Implementations may have practical limits + on the size of `contents` based on memory and storage + available to the workflow runner or other factors. + + If the `loadContents` field of an `InputParameter` or + `OutputParameter` is true, and the input or output File object + `location` is valid, the file must be a UTF-8 text file 64 KiB + or smaller, and the implementation must read the entire + contents of the file and place it in the `contents` field. If + the size of the file is greater than 64 KiB, the + implementation must raise a fatal error. + + +- name: Directory + type: record + docAfter: "#File" + doc: | + Represents a directory to present to a command line tool. + + Directories are represented as objects with `class` of `Directory`. Directory objects have + a number of properties that provide metadata about the directory. + + The `location` property of a Directory is a IRI that uniquely identifies + the directory. Implementations must support the file:// IRI scheme and may + support other schemes such as http://. Alternately to `location`, + implementations must also accept the `path` property on Directory, which + must be a filesystem path available on the same host as the CWL runner (for + inputs) or the runtime environment of a command line tool execution (for + command line tool outputs). + + A Directory object may have a `listing` field. This is a list of File and + Directory objects that are contained in the Directory. For each entry in + `listing`, the `basename` property defines the name of the File or + Subdirectory when staged to disk. If `listing` is not provided, the + implementation must have some way of fetching the Directory listing at + runtime based on the `location` field. + + If a Directory does not have `location`, it is a Directory literal. A + Directory literal must provide `listing`. Directory literals must be + created on disk at runtime as needed. + + The resources in a Directory literal do not need to have any implied + relationship in their `location`. For example, a Directory listing may + contain two files located on different hosts. It is the responsibility of + the runtime to ensure that those files are staged to disk appropriately. + Secondary files associated with files in `listing` must also be staged to + the same Directory. + + When executing a CommandLineTool, Directories must be recursively staged + first and have local values of `path` assigned. + + Directory objects in CommandLineTool output must provide either a + `location` IRI or a `path` property in the context of the tool execution + runtime (local to the compute node, or within the executing container). + + An ExpressionTool may forward file references from input to output by using + the same value for `location`. + + Name conflicts (the same `basename` appearing multiple times in `listing` + or in any entry in `secondaryFiles` in the listing) is a fatal error. + + fields: + - name: class + type: + type: enum + name: Directory_class + symbols: + - cwl:Directory + jsonldPredicate: + _id: "@type" + _type: "@vocab" + doc: Must be `Directory` to indicate this object describes a Directory. + - name: location + type: string? + doc: | + An IRI that identifies the directory resource. This may be a relative + reference, in which case it must be resolved using the base IRI of the + document. The location may refer to a local or remote resource. If + the `listing` field is not set, the implementation must use the + location IRI to retrieve directory listing. If an implementation is + unable to retrieve the directory listing stored at a remote resource (due to + unsupported protocol, access denied, or other issue) it must signal an + error. + + If the `location` field is not provided, the `listing` field must be + provided. The implementation must assign a unique identifier for + the `location` field. + + If the `path` field is provided but the `location` field is not, an + implementation may assign the value of the `path` field to `location`, + then follow the rules above. + jsonldPredicate: + _id: "@id" + _type: "@id" + - name: path + type: string? + doc: | + The local path where the Directory is made available prior to executing a + CommandLineTool. This must be set by the implementation. This field + must not be used in any other context. The command line tool being + executed must be able to access the directory at `path` using the POSIX + `opendir(2)` syscall. + + If the `path` contains [POSIX shell metacharacters](http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_02) + (`|`,`&`, `;`, `<`, `>`, `(`,`)`, `$`,`` ` ``, `\`, `"`, `'`, + ``, ``, and ``) or characters + [not allowed](http://www.iana.org/assignments/idna-tables-6.3.0/idna-tables-6.3.0.xhtml) + for [Internationalized Domain Names for Applications](https://tools.ietf.org/html/rfc6452) + then implementations may terminate the process with a + `permanentFailure`. + jsonldPredicate: + _id: "cwl:path" + _type: "@id" + - name: basename + type: string? + doc: | + The base name of the directory, that is, the name of the file without any + leading directory path. The base name must not contain a slash `/`. + + If not provided, the implementation must set this field based on the + `location` field by taking the final path component after parsing + `location` as an IRI. If `basename` is provided, it is not required to + match the value from `location`. + + When this file is made available to a CommandLineTool, it must be named + with `basename`, i.e. the final component of the `path` field must match + `basename`. + jsonldPredicate: "cwl:basename" + - name: listing + type: + - "null" + - type: array + items: [File, Directory] + doc: | + List of files or subdirectories contained in this directory. The name + of each file or subdirectory is determined by the `basename` field of + each `File` or `Directory` object. It is an error if a `File` shares a + `basename` with any other entry in `listing`. If two or more + `Directory` object share the same `basename`, this must be treated as + equivalent to a single subdirectory with the listings recursively + merged. + jsonldPredicate: + _id: "cwl:listing" + + +- name: CWLObjectType + type: union + names: + - boolean + - int + - long + - float + - double + - string + - File + - Directory + - type: array + items: + - "null" + - CWLObjectType + - type: map + values: + - "null" + - CWLObjectType + doc: | + Generic type representing a valid CWL object. It is used to represent + `default` values passed to CWL `InputParameter` and `WorkflowStepInput` + record fields. + +- name: CWLInputFile + type: map + values: + - "null" + - type: array + items: ProcessRequirement + - CWLObjectType + doc: | + Type representing a valid CWL input file as a `map, CWLObjectType>>`. + jsonldPredicate: + _id: "cwl:inputfile" + _container: "@list" + noLinkCheck: true diff --git a/cwltool/schemas/v1.3.0-dev1/CITATION b/cwltool/schemas/v1.3.0-dev1/CITATION new file mode 100644 index 000000000..526be185e --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/CITATION @@ -0,0 +1,28 @@ +To cite the Common Workflow Language standards in a publication, please use: + +Amstutz, Peter; Crusoe, Michael R; Tijanić, Nebojša; Chapman, Brad; +Chilton, John; Heuer, Michael; Kartashov, Andrey; Kern, John; Leehr, Dan; +Ménager, Hervé; Nedeljkovich, Maya; Scales, Matt; Soiland-Reyes, Stian; +Stojanovic, Luka (2016): Common Workflow Language, v1.0. Standards, +Common Workflow Language working group. https://w3id.org/cwl/v1.0/ +https://doi.org/10.6084/m9.figshare.3115156.v2 + +@data{cwl, + doi = {10.6084/m9.figshare.3115156.v2}, + url = {https://doi.org/10.6084/m9.figshare.3115156.v2}, + author = {Peter Amstutz; Michael R. Crusoe; Nebojša Tijanić; Brad Chapman; +John Chilton; Michael Heuer; Andrey Kartashov; John Kern; Dan Leehr; +Hervé Ménager; Maya Nedeljkovich; Matt Scales; Stian Soiland-Reyes; +Luka Stojanovic + }, + publisher = {Figshare}, + institution = {Common Workflow Language working group}, + title = {Common Workflow Language, v1.0}, + year = {2016} +} + +# Are you editing this file? +# Synchronize any changes made with +# README.md +# and +# https://github.com/common-workflow-language/user_guide/blob/gh-pages/CITATION diff --git a/cwltool/schemas/v1.3.0-dev1/CODE_OF_CONDUCT.md b/cwltool/schemas/v1.3.0-dev1/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..4f0418930 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/CODE_OF_CONDUCT.md @@ -0,0 +1,107 @@ +CWL Code of Conduct +=================== + +The CWL Project is dedicated to providing a harassment-free experience for +everyone. We do not tolerate harassment of participants in any form. + +This code of conduct applies to all CWL Project spaces both online and off: the +Google Group, the Gitter chat room, the Google Hangouts chats, and any other +CWL spaces. Anyone who violates this code of conduct may be sanctioned or +expelled from these spaces at the discretion of the CWL Leadership Team. + +Some CWL Project spaces may have additional rules in place, which will be +made clearly available to participants. Participants are responsible for +knowing and abiding by these rules. + +Harassment includes, but is not limited to: + + - Offensive comments related to gender, gender identity and expression, sexual +orientation, disability, mental illness, neuro(a)typicality, physical +appearance, body size, age, race, or religion. + - Unwelcome comments regarding a person’s lifestyle choices and practices, +including those related to food, health, parenting, drugs, and employment. + - Deliberate misgendering or use of [dead](https://www.quora.com/What-is-deadnaming/answer/Nancy-C-Walker) +or rejected names. + - Gratuitous or off-topic sexual images or behaviour in spaces where they’re not +appropriate. + - Physical contact and simulated physical contact (eg, textual descriptions like +“\*hug\*” or “\*backrub\*”) without consent or after a request to stop. + - Threats of violence. + - Incitement of violence towards any individual, including encouraging a person +to commit suicide or to engage in self-harm. + - Deliberate intimidation. + - Stalking or following. + - Harassing photography or recording, including logging online activity for +harassment purposes. + - Sustained disruption of discussion. + - Unwelcome sexual attention. + - Pattern of inappropriate social contact, such as requesting/assuming +inappropriate levels of intimacy with others + - Continued one-on-one communication after requests to cease. + - Deliberate “outing” of any aspect of a person’s identity without their consent +except as necessary to protect vulnerable people from intentional abuse. + - Publication of non-harassing private communication. + +The CWL Project prioritizes marginalized people’s safety over privileged +people’s comfort. The CWL Leadership Team will not act on complaints regarding: + + - ‘Reverse’ -isms, including ‘reverse racism,’ ‘reverse sexism,’ and ‘cisphobia’ + - Reasonable communication of boundaries, such as “leave me alone,” “go away,” or +“I’m not discussing this with you.” + - Communicating in a [tone](http://geekfeminism.wikia.com/wiki/Tone_argument) +you don’t find congenial + +Reporting +--------- + +If you are being harassed by a member of the CWL Project, notice that someone +else is being harassed, or have any other concerns, please contact the CWL +Leadership Team at leadership@commonwl.org. If person who is harassing +you is on the team, they will recuse themselves from handling your incident. We +will respond as promptly as we can. + +This code of conduct applies to CWL Project spaces, but if you are being +harassed by a member of CWL Project outside our spaces, we still want to +know about it. We will take all good-faith reports of harassment by CWL Project +members, especially the CWL Leadership Team, seriously. This includes harassment +outside our spaces and harassment that took place at any point in time. The +abuse team reserves the right to exclude people from the CWL Project based on +their past behavior, including behavior outside CWL Project spaces and +behavior towards people who are not in the CWL Project. + +In order to protect volunteers from abuse and burnout, we reserve the right to +reject any report we believe to have been made in bad faith. Reports intended +to silence legitimate criticism may be deleted without response. + +We will respect confidentiality requests for the purpose of protecting victims +of abuse. At our discretion, we may publicly name a person about whom we’ve +received harassment complaints, or privately warn third parties about them, if +we believe that doing so will increase the safety of CWL Project members or +the general public. We will not name harassment victims without their +affirmative consent. + +Consequences +------------ + +Participants asked to stop any harassing behavior are expected to comply +immediately. + +If a participant engages in harassing behavior, the CWL Leadership Team may +take any action they deem appropriate, up to and including expulsion from all +CWL Project spaces and identification of the participant as a harasser to other +CWL Project members or the general public. + +This anti-harassment policy is based on the [example policy from the Geek +Feminism wiki](http://geekfeminism.wikia.com/wiki/Community_anti-harassment/Policy), +created by the Geek Feminism community. + +CWL Leadership Team +------------------- + +As a stop gap measure until a more formal governance structure is adopted, the +following individuals make up the leadership of the CWL Project: Peter Amstutz, +John Chilton, Michael R. Crusoe, and Nebojša Tijanić. + +To report an issue with anyone on the team you can escalate to Anton Nekrutenko (Galaxy) +anton AT bx DOT psu DOT edu, C. Titus Brown (UC Davis) ctbrown@ucdavis.edu, or +Brandi Davis-Dusenbery (Seven Bridges Genomics) brandi@sbgenomics.com. diff --git a/cwltool/schemas/v1.3.0-dev1/CONFORMANCE_TESTS.md b/cwltool/schemas/v1.3.0-dev1/CONFORMANCE_TESTS.md new file mode 100644 index 000000000..8e401ac5d --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/CONFORMANCE_TESTS.md @@ -0,0 +1,222 @@ +# Common workflow language conformance test suite + +The conformance tests are intended to test feature coverage of a CWL +implementation. It uses the module cwltest from https://github.com/common-workflow-language/cwltest/. + +## Pre-requisites + +You will need both the `cwltest` Python package and the CWL runner you would like to test installed. + +Installing the `cwltest` Python package using a virtualenv: + +``` +$ python3 -m venv cwltest_env +$ source cwltest_env/bin/activate +$ pip install cwltest +``` + +or via `bioconda` + +``` +$ conda install -c bioconda cwltest +``` + +## Usage + +```bash +$ ./run_test.sh +--- Running conformance test draft-3 on cwl-runner --- +Test [49/49] +All tests passed +``` + + +## Options + +`RUNNER=other-cwl-runner` + +The CWL implementation to be tested. + +### Test Selection + +`--tags` + +A comma separated list of [tags](#tags-for-conformance-tests); only tests with these tags will be tested. +`--tags shell_command` will run all tests with `shell_command` in their `tags` list. + +`-n{test_range}` + +Run only the specific test numbers. `{test_range}` is a comma separated list of +single test numbers and/or numeric ranges. +`-n5-7,15` == only runs the 5th, 6th, 7th, and 15th tests. + +`-N{test_range}` + +Like the lowercase `-n` option, except that the specified tests will not be run. +Can be mixed with the other test selectors: `-n5-7,15 -N6` == only runs the 5th, 7th, and 15th tests, skipping the 6th test. + +`-s{test_names}` + +Run the specific tests according to their `id`s. `{test_names}` is a comma separated list of +test identifiers (the `id` field). `-scl_optional_bindings_provided,stdout_redirect_docker,expression_any_null` +achieves the same effect as `-n5-7,15 -N6` and it will still work if the tests are re-ordered. + +`-S{test_names}` + +Excludes specific tests according to their `id`s. `{test_names}` is a comma separated list of +test identifiers (the `id` field). `--tags shell_command -Sstderr_redirect_shortcut` +will run all tests with `expression_tool` in their `tags` list except the test +with the `id` of `stderr_redirect_shortcut`. + +### Misc + +`EXTRA="--parallel --singularity"` + +Extra options to pass to the CWL runner (check your runner for exact options) + +`-j=4` + +The number of different tests to run at the same time + +`--junit-xml=FILENAME` + +Store results in JUnit XML format using the given FILENAME. + +`--classname=CLASSNAME` + +In the JUnit XML, tag the results with the given CLASSNAME. +Can be useful when multiple test runs are combined to document the differences in `EXTRA=`. + +--- + +For example, to run conformance test 15,16,17, and 20 against the "cwltool" +reference implementation using the `podman` container engine +and parallel execution of workflow steps + +```bash +$ ./run_test.sh RUNNER=cwltool -n15-17,20 EXTRA="--parallel --podman" +Test [15/49] +Test [16/49] +Test [17/49] +Test [20/49] +All tests passed +``` + +## OS X / macOS Notes + +_NOTE_: For running on OSX systems, you'll need to install coreutils via brew. This will add to your +system some needed GNU-like tools like `greadlink`. + +1. If you haven't already, install [brew](http://brew.sh/) package manager in your mac +2. Run `brew install coreutils` + +## Format of the conformance test file + +A single conformance test consist of the path to an input CWL document plus an input CWL object +and the expected outputs (or `should_fail: true` if the test is deliberately broken) + +They are stored in [`conformance_tests.yaml`](https://github.com/common-workflow-language/cwl-v1.2/blob/main/conformance_tests.yaml) +(or a file `$import`ed into that one) + +You can examine [the formal schema of this file](https://github.com/common-workflow-language/cwltest/blob/main/cwltest/cwltest-schema.yml), +or just continue reading here for an explanation. + +The conformance test file is a YAML document: a list of key-value pairs. + +We will use this single entry to explain the format +``` yaml +- doc: Test command line with optional input (missing) + id: cl_optional_inputs_missing + tool: tests/cat1-testcli.cwl + job: tests/cat-job.json + output: + args: [cat, hello.txt] + tags: [ required, command_line_tool ] +``` +- `doc`: A unique, single-line sentence that explain what is being tested. + Will be printed at test execution time, so please don't make it too long! + Additional documentation can go as comments in the CWL document itself. +- `id`: a short list of underscore (`_`) separated words that succinctly identifies and explains the test. +- `tool` the path to the CWL document to run +- `job`: the CWL input object in YAML/JSON format. If there are no inputs then use `tests/empty.json`. +- `output` [the CWL output object expected.](#output-matching) +- `tags`: a yaml list of tag names, see [the list of canonical tags below](#tags-for-conformance-tests). + Must include one or more of the following tags: `command_line_tool`, `expression_tool` or `workflow`. + If the test does not test any optional features, the tag `required` is required. + +Because `conformance_tests.yaml` is a `schema-salad` processed document, [`$import`](https://www.commonwl.org/v1.3/SchemaSalad.html#Import) +can be used to organize the tests into separate files. + +Currently, the main file is too big (over 3400 lines); we are slowly re-organizing it. + +Eventually it would be good to organize the tests so that the test for each optional feature and other logical groups of tests are in their own separate file; +with the supporting CWL documents and their inputs in separate sub-folders of `tests` as well. + +Example: [`- $import: tests/string-interpolation/test-index.yaml`](https://github.com/common-workflow-language/cwl-v1.2/blob/5f27e234b4ca88ed1280dedf9e3391a01de12912/conformance_tests.yaml#L3395) +adds all the entries in [`tests/string-interpolation/test-index.yaml`](https://github.com/common-workflow-language/cwl-v1.2/blob/main/tests/string-interpolation/test-index.yaml) +as entries in the main conformance test file. + +## Output matching + +In each test entry there is an `output` field that contains a mapping of the expected outputs names and their values. + +If a particular value could vary and it doesn't matter to the proper functioning of the test, then it can be represented by the special token `Any`. + +At any level, if there is an extra field, then that will be considered an error. +An exception to this is `class: File` and `class: Directory` objects, the `cwl-runner` under test can add additional fields here without causing a test to fail. +Likewise, if you don't want to test some aspect of a `class: File` or `class: Directory` object (like `nameext`) you can just omit it. + +[According to the CWL standards](https://www.commonwl.org/v1.3/CommandLineTool.html#File), the format of the `location` field in +`class: File` and `class: Directory` is implementation specific and we should not be testing them. +Please remember to use `location: Any` for them. + +Currently, we do [test the contents of the location field in some older tests, but we should stop](https://github.com/common-workflow-language/common-workflow-language/issues/930) +If you are editing those old tests, you may be interested in some special processing for `class: File` and `class: Directory` output objects: +any `location` value specified will succeed if there is either an exact match to the real output, or it matches the end of the real output. +Additionally, for `class: Directory` the location reported by the actual execution will have any trailing forward slash (`/`) trimmed off before comparison. + +Likewise, please do not test the `path` for `class: File` and `class: Directory`. + +## Writing a new conformance test + +To add a new conformance test: +1. Ensure the CWL document you have tests the desired feature or aspect. +2. The `cwlVersion` should be the latest version (`cwlVersion: v1.2`), unless + testing the mixing of versions as in the `tests/mixed-versions` directory. +3. All `CommandLineTool`s need a software container (via `DockerRequirement`) for better reproducibility, preferably under `hints`. + Please limit your container usage to the following: + - `dockerPull: docker.io/alpine:latest` + - `dockerPull: docker.io/bash:4.4` + - `dockerPull: docker.io/debian:stable-slim` + - `dockerPull: docker.io/python:3-slim` +4. Run your test using the CWL reference runner (`cwltool`) or another CWL runner + that shows the correct behavior to collect the output, or confirm that validation/execution fails as expected +5. Add the CWL document and output object to the subdirectory `tests` in this repository. +6. Fill out a new entry in [conformance_tests.yaml](conformance_tests.yaml) following the [format of the conformance test file](#format-of-the-conformance-test-file) +7. Send a pull request to [current staging branch for the next revision of the CWL standards](https://github.com/common-workflow-language/cwl-v1.2/tree/1.2.1_proposed) + with your changes + +## Tags for conformance tests + +Each test in the [conformance_tests.yaml](conformance_tests.yaml) should be tagged with one or more tags. + +1. A `command_line_tool`, `expression_tool` or `workflow` tag to identify whether a CommandLineTool, ExpressionTool + or Workflow is being tested +2. If the test does not test any optional features, the tag `required` +3. The name of any features that are being tested: + 1. `docker` for DockerRequirement + 1. `env_var` for EnvVarRequirement + 1. `format_checking` for checking format requirement annotation on File inputs + 1. `initial_work_dir` for InitialWorkDirRequirements + 1. `inline_javascript` for InlineJavascriptRequirement + 1. `inplace_update` for InplaceUpdateRequirement + 1. `input_object_requirements` for tests that use cwl:requirements in the input object + 1. `multiple_input` for MultipleInputFeatureRequirement + 1. `networkaccess` for NetworkAccess + 1. `resource` for ResourceRequirement + 1. `scatter` for ScatterFeatureRequirement + 1. `schema_def` for SchemaDefRequirement + 1. `shell_command` for ShellCommandRequirement + 1. `step_input` for StepInputExpressionRequirement + 1. `subworkflow` for SubworkflowRequirement + 1. `timelimit` for ToolTimeLimit diff --git a/cwltool/schemas/v1.3.0-dev1/CommandLineTool-standalone.yml b/cwltool/schemas/v1.3.0-dev1/CommandLineTool-standalone.yml new file mode 100644 index 000000000..6f9072cff --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/CommandLineTool-standalone.yml @@ -0,0 +1,2 @@ +- $import: Process.yml +- $import: CommandLineTool.yml diff --git a/cwltool/schemas/v1.3.0-dev1/CommandLineTool.yml b/cwltool/schemas/v1.3.0-dev1/CommandLineTool.yml new file mode 100644 index 000000000..604c668a7 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/CommandLineTool.yml @@ -0,0 +1,1399 @@ +saladVersion: v1.3 +$base: "https://w3id.org/cwl/cwl#" + +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + +$graph: + +- name: CommandLineToolDoc + type: documentation + doc: + - | + # Common Workflow Language (CWL) Command Line Tool Description, v1.3.0-dev1 + + This version: + * https://w3id.org/cwl/v1.3.0-dev1 + + Latest stable version: + * https://w3id.org/cwl/ + - "\n\n" + - {$include: contrib.md} + - "\n\n" + - | + # Abstract + + A Command Line Tool is a non-interactive executable program that reads + some input, performs a computation, and terminates after producing some + output. Command line programs are a flexible unit of code sharing and + reuse, unfortunately the syntax and input/output semantics among command + line programs is extremely heterogeneous. A common layer for describing + the syntax and semantics of programs can reduce this incidental + complexity by providing a consistent way to connect programs together. + This specification defines the Common Workflow Language (CWL) Command + Line Tool Description, a vendor-neutral standard for describing the + syntax and input/output semantics of command line programs. + + - {$include: intro.md} + + - | + ## Introduction to the CWL Command Line Tool standard v1.3.0-dev1 + + This specification represents the latest development version from the + CWL group. + + Documents should use `cwlVersion: v1.3.0-dev1` to make use of new + syntax and features introduced in v1.3.0-dev1. Existing v1.2 documents + should be trivially updatable by changing `cwlVersion`, however + CWL documents that relied on previously undefined or + underspecified behavior may have slightly different behavior in + v1.3.0-dev1. + + ## Changelog for v1.3.0-dev1 + + See also the [CWL Workflow Description, v1.3.0-dev1 changelog](Workflow.html#Changelog). + For other changes since CWL v1.0, see the + [CWL Command Line Tool Description, v1.1 changelog](https://www.commonwl.org/v1.1/CommandLineTool.html#Changelog) + and + [CWL Command Line Tool Description, v1.2.1 changelog](https://www.commonwl.org/v1.2/CommandLineTool.html#Changelog). + + ## Purpose + + Standalone programs are a flexible and interoperable form of code reuse. + Unlike monolithic applications, applications and analysis workflows which + are composed of multiple separate programs can be written in multiple + languages and execute concurrently on multiple hosts. However, POSIX + does not dictate computer-readable grammar or semantics for program input + and output, resulting in extremely diverse command line grammar and + input/output semantics among programs. This is a particular problem in + distributed computing (multi-node compute clusters) and virtualized + environments (such as Docker containers) where it is often necessary to + provision resources such as input files before executing the program. + + Often this gap is filled by hard coding program invocation and + implicitly assuming requirements will be met, or abstracting program + invocation with wrapper scripts or descriptor documents. Unfortunately, + where these approaches are application or platform specific it creates a + significant barrier to reproducibility and portability, as methods + developed for one platform must be manually ported to be used on new + platforms. Similarly, it creates redundant work, as wrappers for popular + tools must be rewritten for each application or platform in use. + + The Common Workflow Language Command Line Tool Description is designed to + provide a common standard description of grammar and semantics for + invoking programs used in data-intensive fields such as Bioinformatics, + Chemistry, Physics, Astronomy, and Statistics. This specification + attempts to define a precise data and execution model for Command Line Tools that + can be implemented on a variety of computing platforms, ranging from a + single workstation to cluster, grid, cloud, and high performance + computing platforms. Details related to execution of these programs not + laid out in this specification are open to interpretation by the computing + platform implementing this specification. + + - {$include: concepts.md} + - {$include: invocation.md} + + +- type: record + name: EnvironmentDef + doc: | + Define an environment variable that will be set in the runtime environment + by the workflow platform when executing the command line tool. May be the + result of executing an expression, such as getting a parameter from input. + fields: + - name: envName + type: string + doc: The environment variable name + - name: envValue + type: [string, Expression] + doc: The environment variable value + +- type: record + name: CommandLineBinding + extends: InputBinding + docParent: "#CommandInputParameter" + doc: | + + When listed under `inputBinding` in the input schema, the term + "value" refers to the corresponding value in the input object. For + binding objects listed in `CommandLineTool.arguments`, the term "value" + refers to the effective value after evaluating `valueFrom`. + + The binding behavior when building the command line depends on the data + type of the value. If there is a mismatch between the type described by + the input schema and the effective value, such as resulting from an + expression evaluation, an implementation must use the data type of the + effective value. + + - **string**: Add `prefix` and the string to the command line. + + - **number**: Add `prefix` and decimal representation to command line. + + - **boolean**: If true, add `prefix` to the command line. If false, add + nothing. + + - **File**: Add `prefix` and the value of + [`File.path`](#File) to the command line. + + - **Directory**: Add `prefix` and the value of + [`Directory.path`](#Directory) to the command line. + + - **array**: If `itemSeparator` is specified, add `prefix` and the join + the array into a single string with `itemSeparator` separating the + items. Otherwise, first add `prefix`, then recursively process + individual elements. + If the array is empty, it does not add anything to command line. + + - **object**: Add `prefix` only, and recursively add object fields for + which `inputBinding` is specified. + + - **null**: Add nothing. + + fields: + - name: position + type: [ "null", int, Expression ] + default: 0 + doc: | + The sorting key. Default position is 0. If a [CWL Parameter Reference](#Parameter_references) + or [CWL Expression](#Expressions_(Optional)) is used and if the + inputBinding is associated with an input parameter, then the value of + `self` will be the value of the input parameter. Input parameter + defaults (as specified by the `InputParameter.default` field) must be + applied before evaluating the expression. Expressions must return a + single value of type int or a null. + - name: prefix + type: string? + doc: "Command line prefix to add before the value." + - name: separate + type: boolean? + default: true + doc: | + If true (default), then the prefix and value must be added as separate + command line arguments; if false, prefix and value must be concatenated + into a single command line argument. + - name: itemSeparator + type: string? + doc: | + Join the array elements into a single string with the elements + separated by `itemSeparator`. + - name: valueFrom + type: + - "null" + - string + - Expression + jsonldPredicate: "cwl:valueFrom" + doc: | + If `valueFrom` is a constant string value, use this as the value and + apply the binding rules above. + + If `valueFrom` is an expression, evaluate the expression to yield the + actual value to use to build the command line and apply the binding + rules above. If the inputBinding is associated with an input + parameter, the value of `self` in the expression will be the value of + the input parameter. Input parameter defaults (as specified by the + `InputParameter.default` field) must be applied before evaluating the + expression. + + If the value of the associated input parameter is `null`, `valueFrom` is + not evaluated and nothing is added to the command line. + + When a binding is part of the `CommandLineTool.arguments` field, + the `valueFrom` field is required. + + - name: shellQuote + type: boolean? + default: true + doc: | + If `ShellCommandRequirement` is in the requirements for the current command, + this controls whether the value is quoted on the command line (default is true). + Use `shellQuote: false` to inject metacharacters for operations such as pipes. + + If `shellQuote` is true or not provided, the implementation must not + permit interpretation of any shell metacharacters or directives. + + +- type: record + name: CommandOutputBinding + extends: LoadContents + doc: | + Describes how to generate an output parameter based on the files produced + by a CommandLineTool. + + The output parameter value is generated by applying these operations in the + following order: + + - glob + - loadContents + - outputEval + - secondaryFiles + fields: + - name: glob + type: + - "null" + - string + - Expression + - type: array + items: string + doc: | + Find files or directories relative to the output directory, using POSIX + glob(3) pathname matching. If an array is provided, find files or + directories that match any pattern in the array. If an expression is + provided, the expression must return a string or an array of strings, + which will then be evaluated as one or more glob patterns. Must only + match and return files/directories which actually exist. + + If the value of glob is a relative path pattern (does not + begin with a slash '/') then it is resolved relative to the + output directory. If the value of the glob is an absolute + path pattern (it does begin with a slash '/') then it must + refer to a path within the output directory. It is an error + if any glob resolves to a path outside the output directory. + Specifically this means globs that resolve to paths outside the output + directory are illegal. + + A glob may match a path within the output directory which is + actually a symlink to another file. In this case, the + expected behavior is for the resulting File/Directory object to take the + `basename` (and corresponding `nameroot` and `nameext`) of the + symlink. The `location` of the File/Directory is implementation + dependent, but logically the File/Directory should have the same content + as the symlink target. Platforms may stage output files/directories to + cloud storage that lack the concept of a symlink. In + this case file content and directories may be duplicated, or (to avoid + duplication) the File/Directory `location` may refer to the symlink + target. + + It is an error if a symlink in the output directory (or any + symlink in a chain of links) refers to any file or directory + that is not under an input or output directory. + + Implementations may shut down a container before globbing + output, so globs and expressions must not assume access to the + container filesystem except for declared input and output. + + - name: outputEval + type: Expression? + doc: | + Evaluate an expression to generate the output value. If + `glob` was specified, the value of `self` must be an array + containing file objects that were matched. If no files were + matched, `self` must be a zero length array; if a single file + was matched, the value of `self` is an array of a single + element. The exit code of the process is + available in the expression as `runtime.exitCode`. + + Additionally, if `loadContents` is true, the file must be a + UTF-8 text file 64 KiB or smaller, and the implementation must + read the entire contents of the file (or file array) and place + it in the `contents` field of the File object for use in + `outputEval`. If the size of the file is greater than 64 KiB, + the implementation must raise a fatal error. + + If a tool needs to return a large amount of structured data to + the workflow, loading the output object from `cwl.output.json` + bypasses `outputEval` and is not subject to the 64 KiB + `loadContents` limit. + +- name: CommandLineBindable + type: record + fields: + inputBinding: + type: CommandLineBinding? + jsonldPredicate: "cwl:inputBinding" + doc: Describes how to turn this object into command line arguments. + +- name: CommandInputRecordField + type: record + extends: [InputRecordField, CommandLineBindable] + specialize: + - specializeFrom: InputRecordSchema + specializeTo: CommandInputRecordSchema + - specializeFrom: InputEnumSchema + specializeTo: CommandInputEnumSchema + - specializeFrom: InputArraySchema + specializeTo: CommandInputArraySchema + - specializeFrom: InputBinding + specializeTo: CommandLineBinding + + +- name: CommandInputRecordSchema + type: record + extends: [InputRecordSchema, CommandInputSchema, CommandLineBindable] + specialize: + - specializeFrom: InputRecordField + specializeTo: CommandInputRecordField + - specializeFrom: InputBinding + specializeTo: CommandLineBinding + + +- name: CommandInputEnumSchema + type: record + extends: [InputEnumSchema, CommandInputSchema, CommandLineBindable] + specialize: + - specializeFrom: InputBinding + specializeTo: CommandLineBinding + + +- name: CommandInputArraySchema + type: record + extends: [InputArraySchema, CommandInputSchema, CommandLineBindable] + specialize: + - specializeFrom: InputRecordSchema + specializeTo: CommandInputRecordSchema + - specializeFrom: InputEnumSchema + specializeTo: CommandInputEnumSchema + - specializeFrom: InputArraySchema + specializeTo: CommandInputArraySchema + - specializeFrom: InputBinding + specializeTo: CommandLineBinding + + +- name: CommandOutputRecordField + type: record + extends: OutputRecordField + specialize: + - specializeFrom: OutputRecordSchema + specializeTo: CommandOutputRecordSchema + - specializeFrom: OutputEnumSchema + specializeTo: CommandOutputEnumSchema + - specializeFrom: OutputArraySchema + specializeTo: CommandOutputArraySchema + fields: + - name: outputBinding + type: CommandOutputBinding? + jsonldPredicate: "cwl:outputBinding" + doc: | + Describes how to generate this output object based on the files + produced by a CommandLineTool + + +- name: CommandOutputRecordSchema + type: record + extends: OutputRecordSchema + specialize: + - specializeFrom: OutputRecordField + specializeTo: CommandOutputRecordField + + +- name: CommandOutputEnumSchema + type: record + extends: OutputEnumSchema + specialize: + - specializeFrom: OutputRecordSchema + specializeTo: CommandOutputRecordSchema + - specializeFrom: OutputEnumSchema + specializeTo: CommandOutputEnumSchema + - specializeFrom: OutputArraySchema + specializeTo: CommandOutputArraySchema + + +- name: CommandOutputArraySchema + type: record + extends: OutputArraySchema + specialize: + - specializeFrom: OutputRecordSchema + specializeTo: CommandOutputRecordSchema + - specializeFrom: OutputEnumSchema + specializeTo: CommandOutputEnumSchema + - specializeFrom: OutputArraySchema + specializeTo: CommandOutputArraySchema + + +- type: record + name: CommandInputParameter + extends: InputParameter + doc: An input parameter for a CommandLineTool. + fields: + - name: type + type: + - CWLType + - stdin + - CommandInputRecordSchema + - CommandInputEnumSchema + - CommandInputArraySchema + - string + - type: array + items: + - CWLType + - CommandInputRecordSchema + - CommandInputEnumSchema + - CommandInputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + - name: inputBinding + type: CommandLineBinding? + doc: | + Describes how to turn the input parameters of a process into + command line arguments. + jsonldPredicate: "cwl:inputBinding" + +- type: record + name: CommandOutputParameter + extends: OutputParameter + doc: An output parameter for a CommandLineTool. + fields: + - name: type + type: + - CWLType + - stdout + - stderr + - CommandOutputRecordSchema + - CommandOutputEnumSchema + - CommandOutputArraySchema + - string + - type: array + items: + - CWLType + - CommandOutputRecordSchema + - CommandOutputEnumSchema + - CommandOutputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + - name: outputBinding + type: CommandOutputBinding? + jsonldPredicate: "cwl:outputBinding" + doc: Describes how to generate this output object based on the files + produced by a CommandLineTool + +- name: stdin + type: enum + symbols: [ "cwl:stdin" ] + docParent: "#CommandOutputParameter" + doc: | + Only valid as a `type` for a `CommandLineTool` input with no + `inputBinding` set. `stdin` must not be specified at the `CommandLineTool` + level. + + The following + ``` + inputs: + an_input_name: + type: stdin + ``` + is equivalent to + ``` + inputs: + an_input_name: + type: File + streamable: true + + stdin: $(inputs.an_input_name.path) + ``` + +- name: stdout + type: enum + symbols: [ "cwl:stdout" ] + docParent: "#CommandOutputParameter" + doc: | + Only valid as a `type` for a `CommandLineTool` output with no + `outputBinding` set. + + The following + ``` + outputs: + an_output_name: + type: stdout + + stdout: a_stdout_file + ``` + is equivalent to + ``` + outputs: + an_output_name: + type: File + streamable: true + outputBinding: + glob: a_stdout_file + + stdout: a_stdout_file + ``` + + If there is no `stdout` name provided, a random filename will be created. + For example, the following + ``` + outputs: + an_output_name: + type: stdout + ``` + is equivalent to + ``` + outputs: + an_output_name: + type: File + streamable: true + outputBinding: + glob: random_stdout_filenameABCDEFG + + stdout: random_stdout_filenameABCDEFG + ``` + + If the `CommandLineTool` contains logically chained commands + (e.g. `echo a && echo b`) `stdout` must include the output of + every command. + + +- name: stderr + type: enum + symbols: [ "cwl:stderr" ] + docParent: "#CommandOutputParameter" + doc: | + Only valid as a `type` for a `CommandLineTool` output with no + `outputBinding` set. + + The following + ``` + outputs: + an_output_name: + type: stderr + + stderr: a_stderr_file + ``` + is equivalent to + ``` + outputs: + an_output_name: + type: File + streamable: true + outputBinding: + glob: a_stderr_file + + stderr: a_stderr_file + ``` + + If there is no `stderr` name provided, a random filename will be created. + For example, the following + ``` + outputs: + an_output_name: + type: stderr + ``` + is equivalent to + ``` + outputs: + an_output_name: + type: File + streamable: true + outputBinding: + glob: random_stderr_filenameABCDEFG + + stderr: random_stderr_filenameABCDEFG + ``` + + +- type: record + name: CommandLineTool + extends: Process + documentRoot: true + specialize: + - specializeFrom: InputParameter + specializeTo: CommandInputParameter + - specializeFrom: OutputParameter + specializeTo: CommandOutputParameter + doc: | + This defines the schema of the CWL Command Line Tool Description document. + + fields: + - name: class + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + type: + type: enum + name: CommandLineTool_class + symbols: + - cwl:CommandLineTool + - name: baseCommand + doc: | + Specifies the program to execute. If an array, the first element of + the array is the command to execute, and subsequent elements are + mandatory command line arguments. The elements in `baseCommand` must + appear before any command line bindings from `inputBinding` or + `arguments`. + + If `baseCommand` is not provided or is an empty array, the first + element of the command line produced after processing `inputBinding` or + `arguments` must be used as the program to execute. + + If the program includes a path separator character it must + be an absolute path, otherwise it is an error. If the program does not + include a path separator, search the `$PATH` variable in the runtime + environment of the workflow runner find the absolute path of the + executable. + type: + - string? + - string[]? + jsonldPredicate: + "_id": "cwl:baseCommand" + "_container": "@list" + - name: arguments + doc: | + Command line bindings which are not directly associated with input + parameters. If the value is a string, it is used as a string literal + argument. If it is an Expression, the result of the evaluation is used + as an argument. + type: + - "null" + - type: array + items: [string, Expression, CommandLineBinding] + jsonldPredicate: + "_id": "cwl:arguments" + "_container": "@list" + - name: stdin + type: ["null", string, Expression] + jsonldPredicate: "https://w3id.org/cwl/cwl#stdin" + doc: | + A path to a file whose contents must be piped into the command's + standard input stream. + - name: stderr + type: ["null", string, Expression] + jsonldPredicate: "https://w3id.org/cwl/cwl#stderr" + doc: | + Capture the command's standard error stream to a file written to + the designated output directory. + + If `stderr` is a string, it specifies the file name to use. + + If `stderr` is an expression, the expression is evaluated and must + return a string with the file name to use to capture stderr. If the + return value is not a string, or the resulting path contains illegal + characters (such as the path separator `/`) it is an error. + - name: stdout + type: ["null", string, Expression] + jsonldPredicate: "https://w3id.org/cwl/cwl#stdout" + doc: | + Capture the command's standard output stream to a file written to + the designated output directory. + + If the `CommandLineTool` contains logically chained commands + (e.g. `echo a && echo b`) `stdout` must include the output of + every command. + + If `stdout` is a string, it specifies the file name to use. + + If `stdout` is an expression, the expression is evaluated and must + return a string with the file name to use to capture stdout. If the + return value is not a string, or the resulting path contains illegal + characters (such as the path separator `/`) it is an error. + - name: successCodes + type: int[]? + doc: | + Exit codes that indicate the process completed successfully. + + If not specified, only exit code 0 is considered success. + + - name: temporaryFailCodes + type: int[]? + doc: | + Exit codes that indicate the process failed due to a possibly + temporary condition, where executing the process with the same + runtime environment and inputs may produce different results. + + If not specified, no exit codes are considered temporary failure. + + - name: permanentFailCodes + type: int[]? + doc: + Exit codes that indicate the process failed due to a permanent logic + error, where executing the process with the same runtime environment and + same inputs is expected to always fail. + + If not specified, all exit codes except 0 are considered permanent failure. + + +- type: record + name: DockerRequirement + extends: ProcessRequirement + doc: | + Indicates that a workflow component should be run in a + [Docker](https://docker.com) or Docker-compatible (such as + [Singularity](https://www.sylabs.io/) and [udocker](https://github.com/indigo-dc/udocker)) container environment and + specifies how to fetch or build the image. + + If a CommandLineTool lists `DockerRequirement` under + `hints` (or `requirements`), it may (or must) be run in the specified Docker + container. + + The platform must first acquire or install the correct Docker image as + specified by `dockerPull`, `dockerImport`, `dockerLoad` or `dockerFile`. + + The platform must execute the tool in the container using `docker run` with + the appropriate Docker image and tool command line. + + The workflow platform may provide input files and the designated output + directory through the use of volume bind mounts. The platform should rewrite + file paths in the input object to correspond to the Docker bind mounted + locations. That is, the platform should rewrite values in the parameter context + such as `runtime.outdir`, `runtime.tmpdir` and others to be valid paths + within the container. The platform must ensure that `runtime.outdir` and + `runtime.tmpdir` are distinct directories. + + When running a tool contained in Docker, the workflow platform must not + assume anything about the contents of the Docker container, such as the + presence or absence of specific software, except to assume that the + generated command line represents a valid command within the runtime + environment of the container. + + A container image may specify an + [ENTRYPOINT](https://docs.docker.com/engine/reference/builder/#entrypoint) + and/or + [CMD](https://docs.docker.com/engine/reference/builder/#cmd). + Command line arguments will be appended after all elements of + ENTRYPOINT, and will override all elements specified using CMD (in + other words, CMD is only used when the CommandLineTool definition + produces an empty command line). + + Use of implicit ENTRYPOINT or CMD are discouraged due to reproducibility + concerns of the implicit hidden execution point (For further discussion, see + [https://doi.org/10.12688/f1000research.15140.1](https://doi.org/10.12688/f1000research.15140.1)). Portable + CommandLineTool wrappers in which use of a container is optional must not rely on ENTRYPOINT or CMD. + CommandLineTools which do rely on ENTRYPOINT or CMD must list `DockerRequirement` in the + `requirements` section. + + ## Interaction with other requirements + + If [EnvVarRequirement](#EnvVarRequirement) is specified alongside a + DockerRequirement, the environment variables must be provided to Docker + using `--env` or `--env-file` and interact with the container's preexisting + environment as defined by Docker. + + fields: + - name: class + type: + type: enum + name: DockerRequirement_class + symbols: + - cwl:DockerRequirement + doc: "Always 'DockerRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: dockerPull + type: string? + doc: | + Specify a Docker image to retrieve using `docker pull`. Can contain the + immutable digest to ensure an exact container is used: + `dockerPull: ubuntu@sha256:45b23dee08af5e43a7fea6c4cf9c25ccf269ee113168c19722f87876677c5cb2` + - name: dockerLoad + type: string? + doc: "Specify an HTTP URL from which to download a Docker image using `docker load`." + - name: dockerFile + type: string? + doc: "Supply the contents of a Dockerfile which will be built using `docker build`." + - name: dockerImport + type: string? + doc: "Provide HTTP URL to download and gunzip a Docker images using `docker import." + - name: dockerImageId + type: string? + doc: | + The image id that will be used for `docker run`. May be a + human-readable image name or the image identifier hash. May be skipped + if `dockerPull` is specified, in which case the `dockerPull` image id + must be used. + - name: dockerOutputDirectory + type: string? + doc: | + Set the designated output directory to a specific location inside the + Docker container. + + +- type: record + name: SoftwareRequirement + extends: ProcessRequirement + doc: | + A list of software packages that should be configured in the environment of + the defined process. + fields: + - name: class + type: + type: enum + name: SoftwareRequirement_class + symbols: + - cwl:SoftwareRequirement + doc: "Always 'SoftwareRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: packages + type: SoftwarePackage[] + doc: "The list of software to be configured." + jsonldPredicate: + mapSubject: package + mapPredicate: specs + +- name: SoftwarePackage + type: record + fields: + - name: package + type: string + doc: | + The name of the software to be made available. If the name is + common, inconsistent, or otherwise ambiguous it should be combined with + one or more identifiers in the `specs` field. + - name: version + type: string[]? + doc: | + The (optional) versions of the software that are known to be + compatible. + - name: specs + type: string[]? + jsonldPredicate: {_type: "@id", noLinkCheck: true} + doc: | + One or more [IRI](https://en.wikipedia.org/wiki/Internationalized_Resource_Identifier)s + identifying resources for installing or enabling the software named in + the `package` field. Implementations may provide resolvers which map + these software identifier IRIs to some configuration action; or they can + use only the name from the `package` field on a best effort basis. + + For example, the IRI https://packages.debian.org/bowtie could + be resolved with `apt-get install bowtie`. The IRI + https://anaconda.org/bioconda/bowtie could be resolved with `conda + install -c bioconda bowtie`. + + IRIs can also be system independent and used to map to a specific + software installation or selection mechanism. + Using [RRID](https://www.identifiers.org/rrid/) as an example: + https://identifiers.org/rrid/RRID:SCR_005476 + could be fulfilled using the above-mentioned Debian or bioconda + package, a local installation managed by [Environment Modules](https://modules.sourceforge.net/), + or any other mechanism the platform chooses. IRIs can also be from + identifier sources that are discipline specific yet still system + independent. As an example, the equivalent [ELIXIR Tools and Data + Service Registry](https://bio.tools) IRI to the previous RRID example is + https://bio.tools/tool/bowtie2/version/2.2.8. + If supported by a given registry, implementations are encouraged to + query these system independent software identifier IRIs directly for + links to packaging systems. + + A site specific IRI can be listed as well. For example, an academic + computing cluster using Environment Modules could list the IRI + `https://hpc.example.edu/modules/bowtie-tbb/1.22` to indicate that + `module load bowtie-tbb/1.1.2` should be executed to make available + `bowtie` version 1.1.2 compiled with the TBB library prior to running + the accompanying Workflow or CommandLineTool. Note that the example IRI + is specific to a particular institution and computing environment as + the Environment Modules system does not have a common namespace or + standardized naming convention. + + This last example is the least portable and should only be used if + mechanisms based off of the `package` field or more generic IRIs are + unavailable or unsuitable. While harmless to other sites, site specific + software IRIs should be left out of shared CWL descriptions to avoid + clutter. + +- name: Dirent + type: record + doc: | + Define a file or subdirectory that must be staged to a particular + place prior to executing the command line tool. May be the result + of executing an expression, such as building a configuration file + from a template. + + Usually files are staged within the [designated output directory](#Runtime_environment). + However, under certain circumstances, files may be staged at + arbitrary locations, see discussion for `entryname`. + + fields: + - name: entryname + type: ["null", string, Expression] + jsonldPredicate: + _id: cwl:entryname + doc: | + The "target" name of the file or subdirectory. If `entry` is + a File or Directory, the `entryname` field overrides the value + of `basename` of the File or Directory object. + + * Required when `entry` evaluates to file contents only + * Optional when `entry` evaluates to a File or Directory object with a `basename` + * Invalid when `entry` evaluates to an array of File or Directory objects. + + If `entryname` is a relative path, it specifies a name within + the designated output directory. A relative path starting + with `../` or that resolves to location above the designated output directory is an error. + + If `entryname` is an absolute path (starts with a slash `/`) + it is an error unless the following conditions are met: + + * `DockerRequirement` is present in `requirements` + * The program is will run inside a software container + where, from the perspective of the program, the root + filesystem is not shared with any other user or + running program. + + In this case, and the above conditions are met, then + `entryname` may specify the absolute path within the container + where the file or directory must be placed. + + - name: entry + type: [string, Expression] + jsonldPredicate: + _id: cwl:entry + doc: | + If the value is a string literal or an expression which evaluates to a + string, a new text file must be created with the string as the file contents. + + If the value is an expression that evaluates to a `File` or + `Directory` object, or an array of `File` or `Directory` + objects, this indicates the referenced file or directory + should be added to the designated output directory prior to + executing the tool. + + If the value is an expression that evaluates to `null`, + nothing is added to the designated output directory, the entry + has no effect. + + If the value is an expression that evaluates to some other + array, number, or object not consisting of `File` or + `Directory` objects, a new file must be created with the value + serialized to JSON text as the file contents. The JSON + serialization behavior should match the behavior of string + interpolation of [Parameter + references](#Parameter_references). + + - name: writable + type: boolean? + default: false + doc: | + If true, the File or Directory (or array of Files or + Directories) declared in `entry` must be writable by the tool. + + Changes to the file or directory must be isolated and not + visible by any other CommandLineTool process. This may be + implemented by making a copy of the original file or + directory. + + Disruptive changes to the referenced file or directory must not + be allowed unless `InplaceUpdateRequirement.inplaceUpdate` is true. + + Default false (files and directories read-only by default). + + A directory marked as `writable: true` implies that all files and + subdirectories are recursively writable as well. + + If `writable` is false, the file may be made available using a + bind mount or file system link to avoid unnecessary copying of + the input file. Command line tools may receive an error on + attempting to rename or delete files or directories that are + not explicitly marked as writable. + + +- name: InitialWorkDirRequirement + type: record + extends: ProcessRequirement + doc: + Define a list of files and subdirectories that must be staged by + the workflow platform prior to executing the command line tool. + + Normally files are staged within the designated output directory. + However, when running inside containers, files may be staged at + arbitrary locations, see discussion for [`Dirent.entryname`](#Dirent). + Together with `DockerRequirement.dockerOutputDirectory` it is + possible to control the locations of both input and output files + when running in containers. + fields: + - name: class + type: + type: enum + name: InitialWorkDirRequirement_class + symbols: + - cwl:InitialWorkDirRequirement + doc: InitialWorkDirRequirement + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: listing + type: + - Expression + - type: array + items: + - "null" + - Dirent + - Expression + - File + - Directory + - type: array + items: + - File + - Directory + jsonldPredicate: + _id: "cwl:listing" + doc: | + The list of files or subdirectories that must be staged prior + to executing the command line tool. + + Return type of each expression must validate as `["null", + File, Directory, Dirent, {type: array, items: [File, + Directory]}]`. + + Each `File` or `Directory` that is returned by an Expression + must be added to the designated output directory prior to + executing the tool. + + Each `Dirent` record that is listed or returned by an + expression specifies a file to be created or staged in the + designated output directory prior to executing the tool. + + Expressions may return null, in which case they have no effect. + + Files or Directories which are listed in the input parameters + and appear in the `InitialWorkDirRequirement` listing must + have their `path` set to their staged location. If the same + File or Directory appears more than once in the + `InitialWorkDirRequirement` listing, the implementation must + choose exactly one value for `path`; how this value is chosen + is undefined. + + +- name: EnvVarRequirement + type: record + extends: ProcessRequirement + doc: | + Define a list of environment variables which will be set in the + execution environment of the tool. See `EnvironmentDef` for details. + fields: + - name: class + type: + type: enum + name: EnvVarRequirement_class + symbols: + - cwl:EnvVarRequirement + doc: "Always 'EnvVarRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: envDef + type: EnvironmentDef[] + doc: The list of environment variables. + jsonldPredicate: + mapSubject: envName + mapPredicate: envValue + + +- type: record + name: ShellCommandRequirement + extends: ProcessRequirement + doc: | + Modify the behavior of CommandLineTool to generate a single string + containing a shell command line. Each item in the `arguments` list must + be joined into a string separated by single spaces and quoted to prevent + interpretation by the shell, unless `CommandLineBinding` for that argument + contains `shellQuote: false`. If `shellQuote: false` is specified, the + argument is joined into the command string without quoting, which allows + the use of shell metacharacters such as `|` for pipes. + fields: + - name: class + type: + type: enum + name: ShellCommandRequirement_class + symbols: + - cwl:ShellCommandRequirement + doc: "Always 'ShellCommandRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + + +- type: record + name: ResourceRequirement + extends: ProcessRequirement + doc: | + Specify basic hardware resource requirements. + + "min" is the minimum amount of a resource that must be reserved to + schedule a job. If "min" cannot be satisfied, the job should not + be run. + + "max" is the maximum amount of a resource that the job shall be + allocated. If a node has sufficient resources, multiple jobs may + be scheduled on a single node provided each job's "max" resource + requirements are met. If a job attempts to exceed its resource + allocation, an implementation may deny additional resources, which + may result in job failure. + + If both "min" and "max" are specified, an implementation may + choose to allocate any amount between "min" and "max", with the + actual allocation provided in the `runtime` object. + + If "min" is specified but "max" is not, then "max" == "min" + If "max" is specified by "min" is not, then "min" == "max". + + It is an error if max < min. + + It is an error if the value of any of these fields is negative. + + If neither "min" nor "max" is specified for a resource, use the default values below. + + fields: + - name: class + type: + type: enum + name: ResourceRequirement_class + symbols: + - cwl:ResourceRequirement + doc: "Always 'ResourceRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: coresMin + type: ["null", int, long, float, Expression] + # default: 1 + # Don't set the default here, we need to be able to distinguish between + # a value not provided and the textual default to be able to implement + # the rules written in the ResourceRequirement doc above + doc: | + Minimum reserved number of CPU cores (default is 1). + + May be a fractional value to indicate to a scheduling + algorithm that one core can be allocated to multiple + jobs. For example, a value of 0.25 indicates that up to 4 + jobs may run in parallel on 1 core. A value of 1.25 means + that up to 3 jobs can run on a 4 core system (4/1.25 ≈ 3). + + Processes can only share a core allocation if the sum of each + of their `ramMax`, `tmpdirMax`, and `outdirMax` requests also + do not exceed the capacity of the node. + + Processes sharing a core must have the same level of isolation + (typically a container or VM) that they would normally have. + + The reported number of CPU cores reserved for the process, + which is available to expressions on the CommandLineTool as + `runtime.cores`, must be a non-zero integer, and may be + calculated by rounding up the cores request to the next whole + number. + + Scheduling systems may allocate fractional CPU resources by + setting quotas or scheduling weights. Scheduling systems that + do not support fractional CPUs may round up the request to the + next whole number. + + - name: coresMax + type: ["null", int, long, float, Expression] + doc: | + Maximum reserved number of CPU cores. + + See `coresMin` for discussion about fractional CPU requests. + + - name: ramMin + type: ["null", int, long, float, Expression] + # default: 256 + # Don't set the default here, we need to be able to distinguish between + # a value not provided and the textual default to be able to implement + # the rules written in the ResourceRequirement doc above. + # Consumers of ResourceRequirement must apply the default value algorithm + # themselves and not rely on schema-salad rules + doc: | + Minimum reserved RAM in mebibytes (2**20) (default is 256) + + May be a fractional value. If so, the actual RAM request must + be rounded up to the next whole number. The reported amount of + RAM reserved for the process, which is available to + expressions on the CommandLineTool as `runtime.ram`, must be a + non-zero integer. + + - name: ramMax + type: ["null", int, long, float, Expression] + doc: | + Maximum reserved RAM in mebibytes (2**20) + + See `ramMin` for discussion about fractional RAM requests. + + - name: tmpdirMin + type: ["null", int, long, float, Expression] + # default: 1024 + # Don't set the default here, we need to be able to distinguish between + # a value not provided and the textual default to be able to implement + # the rules written in the ResourceRequirement doc above + # Consumers of ResourceRequirement must apply the default value algorithm + # themselves and not rely on schema-salad rules + doc: | + Minimum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) (default is 1024) + + May be a fractional value. If so, the actual storage request + must be rounded up to the next whole number. The reported + amount of storage reserved for the process, which is available + to expressions on the CommandLineTool as `runtime.tmpdirSize`, + must be a non-zero integer. + + - name: tmpdirMax + type: ["null", int, long, float, Expression] + doc: | + Maximum reserved filesystem based storage for the designated temporary directory, in mebibytes (2**20) + + See `tmpdirMin` for discussion about fractional storage requests. + + - name: outdirMin + type: ["null", int, long, float, Expression] + # default: 1024 + # Don't set the default here, we need to be able to distinguish between + # a value not provided and the textual default to be able to implement + # the rules written in the ResourceRequirement doc above + # Consumers of ResourceRequirement must apply the default value algorithm + # themselves and not rely on schema-salad rules + doc: | + Minimum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) (default is 1024) + + May be a fractional value. If so, the actual storage request + must be rounded up to the next whole number. The reported + amount of storage reserved for the process, which is available + to expressions on the CommandLineTool as `runtime.outdirSize`, + must be a non-zero integer. + + - name: outdirMax + type: ["null", int, long, float, Expression] + doc: | + Maximum reserved filesystem based storage for the designated output directory, in mebibytes (2**20) + + See `outdirMin` for discussion about fractional storage requests. + + +- type: record + name: WorkReuse + extends: ProcessRequirement + doc: | + For implementations that support reusing output from past work (on + the assumption that same code and same input produce same + results), control whether to enable or disable the reuse behavior + for a particular tool or step (to accommodate situations where that + assumption is incorrect). A reused step is not executed but + instead returns the same output as the original execution. + + If `WorkReuse` is not specified, correct tools should assume it + is enabled by default. + fields: + - name: class + type: + type: enum + name: WorkReuse_class + symbols: + - cwl:WorkReuse + doc: "Always 'WorkReuse'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: enableReuse + type: [boolean, Expression] + default: true + + +- type: record + name: NetworkAccess + extends: ProcessRequirement + doc: | + Indicate whether a process requires outgoing IPv4/IPv6 network + access. Choice of IPv4 or IPv6 is implementation and site + specific, correct tools must support both. + + If `networkAccess` is false or not specified, tools must not + assume network access, except for localhost (the loopback device). + + If `networkAccess` is true, the tool must be able to make outgoing + connections to network resources. Resources may be on a private + subnet or the public Internet. However, implementations and sites + may apply their own security policies to restrict what is + accessible by the tool. + + Enabling network access does not imply a publicly routable IP + address or the ability to accept inbound connections. + + fields: + - name: class + type: + type: enum + name: NetworkAccess_class + symbols: + - cwl:NetworkAccess + doc: "Always 'NetworkAccess'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: networkAccess + type: [boolean, Expression] + +- name: InplaceUpdateRequirement + type: record + extends: cwl:ProcessRequirement + doc: | + + If `inplaceUpdate` is true, then an implementation supporting this + feature may permit tools to directly update files with `writable: + true` in InitialWorkDirRequirement. That is, as an optimization, + files may be destructively modified in place as opposed to copied + and updated. + + An implementation must ensure that only one workflow step may + access a writable file at a time. It is an error if a file which + is writable by one workflow step file is accessed (for reading or + writing) by any other workflow step running independently. + However, a file which has been updated in a previous completed + step may be used as input to multiple steps, provided it is + read-only in every step. + + Workflow steps which modify a file must produce the modified file + as output. Downstream steps which further process the file must + use the output of previous steps, and not refer to a common input + (this is necessary for both ordering and correctness). + + Workflow authors should provide this in the `hints` section. The + intent of this feature is that workflows produce the same results + whether or not InplaceUpdateRequirement is supported by the + implementation, and this feature is primarily available as an + optimization for particular environments. + + Users and implementers should be aware that workflows that + destructively modify inputs may not be repeatable or reproducible. + In particular, enabling this feature implies that WorkReuse should + not be enabled. + + fields: + class: + type: + type: enum + name: InplaceUpdateRequirement_class + symbols: + - cwl:InplaceUpdateRequirement + doc: "Always 'InplaceUpdateRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + inplaceUpdate: + type: boolean + +- type: record + name: ToolTimeLimit + extends: ProcessRequirement + doc: | + Set an upper limit on the execution time of a CommandLineTool. + A CommandLineTool whose execution duration exceeds the time + limit may be preemptively terminated and considered failed. + May also be used by batch systems to make scheduling decisions. + The execution duration excludes external operations, such as + staging of files, pulling a docker image etc, and only counts + wall-time for the execution of the command line itself. + fields: + - name: class + type: + type: enum + name: ToolTimeLimit_class + symbols: + - cwl:ToolTimeLimit + doc: "Always 'ToolTimeLimit'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: timelimit + type: [int, long, Expression] + doc: | + The time limit, in seconds. A time limit of zero means no + time limit. Negative time limits are an error. diff --git a/cwltool/schemas/v1.3.0-dev1/CommonWorkflowLanguage.yml b/cwltool/schemas/v1.3.0-dev1/CommonWorkflowLanguage.yml new file mode 100644 index 000000000..d5fdb9e02 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/CommonWorkflowLanguage.yml @@ -0,0 +1,12 @@ +saladVersion: v1.3 +$base: "https://w3id.org/cwl/cwl#" + +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + sld: "https://w3id.org/cwl/salad#" + +$graph: + +- $import: Process.yml +- $import: CommandLineTool.yml +- $import: Workflow.yml diff --git a/cwltool/schemas/v1.3.0-dev1/LICENSE.txt b/cwltool/schemas/v1.3.0-dev1/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cwltool/schemas/v1.3.0-dev1/Operation.yml b/cwltool/schemas/v1.3.0-dev1/Operation.yml new file mode 100644 index 000000000..5cc571a37 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/Operation.yml @@ -0,0 +1,94 @@ +saladVersion: v1.3 +$base: "https://w3id.org/cwl/cwl#" + +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + +$graph: + +- name: OperationInputParameter + type: record + extends: InputParameter + docParent: "#Operation" + doc: | + Describe an input parameter of an operation. + fields: + - name: type + type: + - CWLType + - InputRecordSchema + - InputEnumSchema + - InputArraySchema + - string + - type: array + items: + - CWLType + - InputRecordSchema + - InputEnumSchema + - InputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + +- name: OperationOutputParameter + type: record + extends: OutputParameter + docParent: "#Operation" + doc: | + Describe an output parameter of an operation. + fields: + - name: type + type: + - CWLType + - OutputRecordSchema + - OutputEnumSchema + - OutputArraySchema + - string + - type: array + items: + - CWLType + - OutputRecordSchema + - OutputEnumSchema + - OutputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + +- type: record + name: Operation + extends: Process + documentRoot: true + specialize: + - specializeFrom: InputParameter + specializeTo: OperationInputParameter + - specializeFrom: OutputParameter + specializeTo: OperationOutputParameter + doc: | + This record describes an abstract operation. It is a potential + step of a workflow that has not yet been bound to a concrete + implementation. It specifies an input and output signature, but + does not provide enough information to be executed. An + implementation (or other tooling) may provide a means of binding + an Operation to a concrete process (such as Workflow, + CommandLineTool, or ExpressionTool) with a compatible signature. + + fields: + - name: class + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + type: + type: enum + name: Operation_class + symbols: + - cwl:Operation diff --git a/cwltool/schemas/v1.3.0-dev1/Process.yml b/cwltool/schemas/v1.3.0-dev1/Process.yml new file mode 100644 index 000000000..3a1f3a22c --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/Process.yml @@ -0,0 +1,677 @@ +saladVersion: v1.3 +$base: "https://w3id.org/cwl/cwl#" + +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + sld: "https://w3id.org/cwl/salad#" + rdfs: "http://www.w3.org/2000/01/rdf-schema#" + +$graph: + +- name: "Common Workflow Language, v1.3.0-dev1" + type: documentation + doc: {$include: concepts.md} + +- $import: "salad/schema_salad/metaschema/metaschema_base.yml" + +- $import: Base.yml + +- name: BaseTypesDoc + type: documentation + doc: | + ## Base types + docChild: + - "#CWLType" + - "#Process" + +- type: enum + name: CWLVersion + doc: "Version symbols for published CWL document versions." + symbols: + - cwl:draft-2 + - cwl:draft-3.dev1 + - cwl:draft-3.dev2 + - cwl:draft-3.dev3 + - cwl:draft-3.dev4 + - cwl:draft-3.dev5 + - cwl:draft-3 + - cwl:draft-4.dev1 + - cwl:draft-4.dev2 + - cwl:draft-4.dev3 + - cwl:v1.0.dev4 + - cwl:v1.0 + - cwl:v1.1.0-dev1 # a dash is required by the semver 2.0 rules + - cwl:v1.1 + - cwl:v1.2.0-dev1 + - cwl:v1.2.0-dev2 + - cwl:v1.2.0-dev3 + - cwl:v1.2.0-dev4 + - cwl:v1.2.0-dev5 + - cwl:v1.2 + - cwl:v1.3.0-dev1 + +- name: Labeled + type: record + abstract: true + fields: + - name: label + type: + - "null" + - string + jsonldPredicate: "rdfs:label" + doc: "A short, human-readable label of this object." + + +- name: Identified + type: record + abstract: true + fields: + - name: id + type: string? + jsonldPredicate: "@id" + doc: "The unique identifier for this object." + + +- name: LoadListingEnum + type: enum + symbols: [no_listing, shallow_listing, deep_listing] + doc: + - | + Specify the desired behavior for loading the `listing` field of + a Directory object for use by expressions. + - "no_listing: Do not load the directory listing." + - "shallow_listing: Only load the top level listing, do not recurse into subdirectories." + - "deep_listing: Load the directory listing and recursively load all subdirectories as well." + + +- name: LoadContents + type: record + abstract: true + fields: + - name: loadContents + type: boolean? + jsonldPredicate: "cwl:loadContents" + doc: | + Only valid when `type: File` or is an array of `items: File`. + + If true, the file (or each file in the array) must be a UTF-8 + text file 64 KiB or smaller, and the implementation must read + the entire contents of the file (or file array) and place it + in the `contents` field of the File object for use by + expressions. If the size of the file is greater than 64 KiB, + the implementation must raise a fatal error. + + - name: loadListing + type: LoadListingEnum? + jsonldPredicate: "cwl:loadListing" + doc: | + Only valid when `type: Directory` or is an array of `items: Directory`. + + Specify the desired behavior for loading the `listing` field of + a Directory object for use by expressions. + + The order of precedence for loadListing is: + + 1. `loadListing` on an individual parameter + 2. Inherited from `LoadListingRequirement` + 3. By default: `no_listing` + +- name: FieldBase + type: record + extends: Labeled + abstract: true + fields: + secondaryFiles: + type: + - "null" + - SecondaryFileSchema + - type: array + items: SecondaryFileSchema + jsonldPredicate: + _id: "cwl:secondaryFiles" + secondaryFilesDSL: true + doc: | + Only valid when `type: File` or is an array of `items: File`. + + Provides a pattern or expression specifying files or + directories that should be included alongside the primary + file. Secondary files may be required or optional. When not + explicitly specified, secondary files specified for `inputs` + are required and `outputs` are optional. An implementation + must include matching Files and Directories in the + `secondaryFiles` property of the primary file. These Files + and Directories must be transferred and staged alongside the + primary file. An implementation may fail workflow execution + if a required secondary file does not exist. + + If the value is an expression, the value of `self` in the expression + must be the primary input or output File object to which this binding + applies. The `basename`, `nameroot` and `nameext` fields must be + present in `self`. For `CommandLineTool` outputs the `path` field must + also be present. The expression must return a filename string relative + to the path to the primary File, a File or Directory object with either + `path` or `location` and `basename` fields set, or an array consisting + of strings or File or Directory objects. It is legal to reference an + unchanged File or Directory object taken from input as a secondaryFile. + The expression may return "null" in which case there is no secondaryFile + from that expression. + + To work on non-filename-preserving storage systems, portable tool + descriptions should avoid constructing new values from `location`, but + should construct relative references using `basename` or `nameroot` + instead. + + If a value in `secondaryFiles` is a string that is not an expression, + it specifies that the following pattern should be applied to the path + of the primary file to yield a filename relative to the primary File: + + 1. If string ends with `?` character, remove the last `?` and mark + the resulting secondary file as optional. + 2. If string begins with one or more caret `^` characters, for each + caret, remove the last file extension from the path (the last + period `.` and all following characters). If there are no file + extensions, the path is unchanged. + 3. Append the remainder of the string to the end of the file path. + + streamable: + type: boolean? + default: false + doc: | + Only valid when `type: File` or is an array of `items: File`. + + A value of `true` indicates that the file is read or written + sequentially without seeking. An implementation may use this flag to + indicate whether it is valid to stream file contents using a named + pipe. Default: `false`. + + +- name: InputFormat + type: record + abstract: true + fields: + format: + type: + - "null" + - string + - type: array + items: string + - Expression + jsonldPredicate: + _id: cwl:format + _type: "@id" + identity: true + noLinkCheck: true + doc: | + Only valid when `type: File` or is an array of `items: File`. + + This must be one or more IRIs of concept nodes + that represents file formats which are allowed as input to this + parameter, preferably defined within an ontology. If no ontology is + available, file formats may be tested by exact match. + + +- name: OutputFormat + type: record + abstract: true + fields: + format: + type: + - "null" + - string + - Expression + jsonldPredicate: + _id: cwl:format + _type: "@id" + identity: true + noLinkCheck: true + doc: | + Only valid when `type: File` or is an array of `items: File`. + + This is the file format that will be assigned to the output + File object. + + +- name: Parameter + type: record + extends: [FieldBase, sld:Documented, Identified] + abstract: true + doc: | + Define an input or output parameter to a process. + + +- type: enum + name: Expression + doc: | + 'Expression' is not a real type. It indicates that a field must allow + runtime parameter references. If [InlineJavascriptRequirement](#InlineJavascriptRequirement) + is declared and supported by the platform, the field must also allow + Javascript expressions. + symbols: + - cwl:ExpressionPlaceholder + + +- name: InputBinding + type: record + fields: + - name: loadContents + type: + - "null" + - boolean + jsonldPredicate: "cwl:loadContents" + doc: | + Use of `loadContents` in `InputBinding` is deprecated. + Preserved for v1.0 backwards compatibility. Will be removed in + CWL v2.0. Use `InputParameter.loadContents` instead. + +- name: IOSchema + extends: [Labeled, sld:Documented] + type: record + abstract: true + fields: + - name: name + type: string? + jsonldPredicate: "@id" + doc: "The identifier for this type" + +- name: InputSchema + extends: [IOSchema] + type: record + abstract: true + +- name: OutputSchema + extends: [IOSchema] + type: record + abstract: true + + +- name: InputRecordField + type: record + extends: [CWLRecordField, FieldBase, InputFormat, LoadContents] + specialize: + - specializeFrom: CWLRecordSchema + specializeTo: InputRecordSchema + - specializeFrom: "sld:EnumSchema" + specializeTo: InputEnumSchema + - specializeFrom: CWLArraySchema + specializeTo: InputArraySchema + - specializeFrom: "sld:PrimitiveType" + specializeTo: CWLType + + +- name: InputRecordSchema + type: record + extends: [CWLRecordSchema, InputSchema] + specialize: + - specializeFrom: CWLRecordField + specializeTo: InputRecordField + + +- name: InputEnumSchema + type: record + extends: ["sld:EnumSchema", InputSchema] + + +- name: InputArraySchema + type: record + extends: [CWLArraySchema, InputSchema] + specialize: + - specializeFrom: CWLRecordSchema + specializeTo: InputRecordSchema + - specializeFrom: "sld:EnumSchema" + specializeTo: InputEnumSchema + - specializeFrom: CWLArraySchema + specializeTo: InputArraySchema + - specializeFrom: "sld:PrimitiveType" + specializeTo: CWLType + + +- name: OutputRecordField + type: record + extends: [CWLRecordField, FieldBase, OutputFormat] + specialize: + - specializeFrom: CWLRecordSchema + specializeTo: OutputRecordSchema + - specializeFrom: "sld:EnumSchema" + specializeTo: OutputEnumSchema + - specializeFrom: CWLArraySchema + specializeTo: OutputArraySchema + - specializeFrom: "sld:PrimitiveType" + specializeTo: CWLType + + +- name: OutputRecordSchema + type: record + extends: [CWLRecordSchema, "#OutputSchema"] + docParent: "#OutputParameter" + specialize: + - specializeFrom: CWLRecordField + specializeTo: OutputRecordField + + +- name: OutputEnumSchema + type: record + extends: ["sld:EnumSchema", OutputSchema] + docParent: "#OutputParameter" + + +- name: OutputArraySchema + type: record + extends: [CWLArraySchema, OutputSchema] + docParent: "#OutputParameter" + specialize: + - specializeFrom: CWLRecordSchema + specializeTo: OutputRecordSchema + - specializeFrom: "sld:EnumSchema" + specializeTo: OutputEnumSchema + - specializeFrom: CWLArraySchema + specializeTo: OutputArraySchema + - specializeFrom: "sld:PrimitiveType" + specializeTo: CWLType + + +- name: InputParameter + type: record + abstract: true + extends: [Parameter, InputFormat, LoadContents] + fields: + - name: default + type: CWLObjectType? + jsonldPredicate: + _id: "sld:default" + _container: "@list" + noLinkCheck: true + doc: | + The default value to use for this parameter if the parameter is missing + from the input object, or if the value of the parameter in the input + object is `null`. Default values are applied before evaluating expressions + (e.g. dependent `valueFrom` fields). + + +- name: OutputParameter + type: record + extends: [Parameter, OutputFormat] + abstract: true + + +- type: record + name: ProcessRequirement + abstract: true + doc: | + A process requirement declares a prerequisite that may or must be fulfilled + before executing a process. See [`Process.hints`](#process) and + [`Process.requirements`](#process). + + Process requirements are the primary mechanism for specifying extensions to + the CWL core specification. + + +- type: record + name: Process + extends: [Identified, Labeled, sld:Documented] + abstract: true + doc: | + + The base executable type in CWL is the `Process` object defined by the + document. Note that the `Process` object is abstract and cannot be + directly executed. + + fields: + - name: id + type: string? + jsonldPredicate: "@id" + doc: | + The unique identifier for this object. + + Only useful for `$graph` at `Process` level. Should not be exposed + to users in graphical or terminal user interfaces. + - name: inputs + type: + type: array + items: InputParameter + jsonldPredicate: + _id: "cwl:inputs" + mapSubject: id + mapPredicate: type + doc: | + Defines the input parameters of the process. The process is ready to + run when all required input parameters are associated with concrete + values. Input parameters include a schema for each parameter which is + used to validate the input object. It may also be used to build a user + interface for constructing the input object. + + When accepting an input object, all input parameters must have a value. + If an input parameter is missing from the input object, it must be + assigned a value of `null` (or the value of `default` for that + parameter, if provided) for the purposes of validation and evaluation + of expressions. + + - name: outputs + type: + type: array + items: OutputParameter + jsonldPredicate: + _id: "cwl:outputs" + mapSubject: id + mapPredicate: type + doc: | + Defines the parameters representing the output of the process. May be + used to generate and/or validate the output object. + - name: requirements + type: ProcessRequirement[]? + jsonldPredicate: + _id: "cwl:requirements" + mapSubject: class + doc: | + Declares requirements that apply to either the runtime environment or the + workflow engine that must be met in order to execute this process. If + an implementation cannot satisfy all requirements, or a requirement is + listed which is not recognized by the implementation, it is a fatal + error and the implementation must not attempt to run the process, + unless overridden at user option. + - name: hints + type: + - "null" + - type: array + items: [ProcessRequirement, Any] + doc: | + Declares hints applying to either the runtime environment or the + workflow engine that may be helpful in executing this process. It is + not an error if an implementation cannot satisfy all hints, however + the implementation may report a warning. + jsonldPredicate: + _id: cwl:hints + noLinkCheck: true + mapSubject: class + - name: cwlVersion + type: CWLVersion? + doc: | + CWL document version. Always required at the document root. Not + required for a Process embedded inside another Process. + jsonldPredicate: + "_id": "cwl:cwlVersion" + "_type": "@vocab" + - name: intent + type: string[]? + jsonldPredicate: + _type: "@id" + identity: true + doc: | + An identifier for the type of computational operation, of this Process. + Especially useful for [`Operation`](Workflow.html#Operation), but can also be used for + [`CommandLineTool`](CommandLineTool.html#CommandLineTool), + [`Workflow`](Workflow.html#Workflow), or [ExpressionTool](Workflow.html#ExpressionTool). + + If provided, then this must be an IRI of a concept node that + represents the type of operation, preferably defined within an ontology. + + For example, in the domain of bioinformatics, one can use an IRI from + the EDAM Ontology's [Operation concept nodes](http://edamontology.org/operation_0004), + like [Alignment](http://edamontology.org/operation_2928), + or [Clustering](http://edamontology.org/operation_3432); or a more + specific Operation concept like + [Split read mapping](http://edamontology.org/operation_3199). + +- name: InlineJavascriptRequirement + type: record + extends: ProcessRequirement + doc: | + Indicates that the workflow platform must support inline Javascript expressions. + If this requirement is not present, the workflow platform must not perform expression + interpolation. + fields: + - name: class + type: + type: enum + name: InlineJavascriptRequirement_class + symbols: + - cwl:InlineJavascriptRequirement + doc: "Always 'InlineJavascriptRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: expressionLib + type: string[]? + doc: | + Additional code fragments that will also be inserted + before executing the expression code. Allows for function definitions that may + be called from CWL expressions. + + +- name: CommandInputSchema + type: record + abstract: true + +- name: SchemaDefRequirement + type: record + extends: ProcessRequirement + doc: | + This field consists of an array of type definitions which must be used when + interpreting the `inputs` and `outputs` fields. When a `type` field + contains a IRI, the implementation must check if the type is defined in + `schemaDefs` and use that definition. If the type is not found in + `schemaDefs`, it is an error. The entries in `schemaDefs` must be + processed in the order listed such that later schema definitions may refer + to earlier schema definitions. + + - **Type definitions are allowed for `enum` and `record` types only.** + - Type definitions may be shared by defining them in a file and then + `$include`-ing them in the `types` field. + - A file can contain a list of type definitions + + fields: + - name: class + type: + type: enum + name: SchemaDefRequirement_class + symbols: + - cwl:SchemaDefRequirement + doc: "Always 'SchemaDefRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + - name: types + type: + type: array + items: CommandInputSchema + doc: The list of type definitions. + +- name: SecondaryFileSchema + type: record + fields: + - name: pattern + type: + - string + - Expression + doc: | + Provides a pattern or expression specifying files or directories that + should be included alongside the primary file. + + If the value is an expression, the value of `self` in the + expression must be the primary input or output File object to + which this binding applies. The `basename`, `nameroot` and + `nameext` fields must be present in `self`. For + `CommandLineTool` inputs the `location` field must also be + present. For `CommandLineTool` outputs the `path` field must + also be present. If secondary files were included on an input + File object as part of the Process invocation, they must also + be present in `secondaryFiles` on `self`. + + The expression must return either: a filename string relative + to the path to the primary File, a File or Directory object + (`class: File` or `class: Directory`) with either `location` + (for inputs) or `path` (for outputs) and `basename` fields + set, or an array consisting of strings or File or Directory + objects as previously described. + + It is legal to use `location` from a File or Directory object + passed in as input, including `location` from secondary files + on `self`. If an expression returns a File object with the + same `location` but a different `basename` as a secondary file + that was passed in, the expression result takes precedence. + Setting the basename with an expression this way affects the + `path` where the secondary file will be staged to in the + CommandLineTool. + + The expression may return "null" in which case there is no + secondary file from that expression. + + To work on non-filename-preserving storage systems, portable + tool descriptions should treat `location` as an + [opaque identifier](#opaque-strings) and avoid constructing new + values from `location`, but should construct relative references + using `basename` or `nameroot` instead, or propagate `location` + from defined inputs. + + If a value in `secondaryFiles` is a string that is not an expression, + it specifies that the following pattern should be applied to the path + of the primary file to yield a filename relative to the primary File: + + 1. If string ends with `?` character, remove the last `?` and mark + the resulting secondary file as optional. + 2. If string begins with one or more caret `^` characters, for each + caret, remove the last file extension from the path (the last + period `.` and all following characters). If there are no file + extensions, the path is unchanged. + 3. Append the remainder of the string to the end of the file path. + - name: required + type: ["null", boolean, Expression] + doc: | + An implementation must not fail workflow execution if `required` is + set to `false` and the expected secondary file does not exist. + Default value for `required` field is `true` for secondary files on + input and `false` for secondary files on output. + doc: | + Secondary files are specified using the following micro-DSL for secondary files: + + * If the value is a string, it is transformed to an object with two fields + `pattern` and `required` + * By default, the value of `required` is `null` + (this indicates default behavior, which may be based on the context) + * If the value ends with a question mark `?` the question mark is + stripped off and the value of the field `required` is set to `False` + * The remaining value is assigned to the field `pattern` + + For implementation details and examples, please see + [this section](SchemaSalad.html#Domain_Specific_Language_for_secondary_files) + in the Schema Salad specification. + +- name: LoadListingRequirement + type: record + extends: ProcessRequirement + doc: | + Specify the desired behavior for loading the `listing` field of + a Directory object for use by expressions. + fields: + class: + type: + type: enum + name: LoadListingRequirement_class + symbols: + - cwl:LoadListingRequirement + doc: "Always 'LoadListingRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + loadListing: + type: LoadListingEnum? + jsonldPredicate: "cwl:loadListing" diff --git a/cwltool/schemas/v1.3.0-dev1/README.md b/cwltool/schemas/v1.3.0-dev1/README.md new file mode 100644 index 000000000..2cb4eef77 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/README.md @@ -0,0 +1,5 @@ +Hello! + +This repo holds the `v1.3.0-dev1` in-development version of the Common Workflow Language standard and the history of its development. + +You can render this draft specification using the `render.bash` script. diff --git a/cwltool/schemas/v1.3.0-dev1/Workflow.yml b/cwltool/schemas/v1.3.0-dev1/Workflow.yml new file mode 100644 index 000000000..f83575b2a --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/Workflow.yml @@ -0,0 +1,911 @@ +saladVersion: v1.3 +$base: "https://w3id.org/cwl/cwl#" + +$namespaces: + cwl: "https://w3id.org/cwl/cwl#" + rdfs: "http://www.w3.org/2000/01/rdf-schema#" + +$graph: + +- name: "WorkflowDoc" + type: documentation + doc: + - | + # Common Workflow Language (CWL) Workflow Description, v1.3.0-dev1 + + This version: + * https://w3id.org/cwl/v1.3.0-dev1 + + Latest stable version: + * https://w3id.org/cwl/ + - "\n\n" + - {$include: contrib.md} + - "\n\n" + - | + # Abstract + + This specification defines the Common Workflow Language (CWL) + Workflow description, a vendor-neutral standard for representing + analysis tasks where a sequence of operations are described + using a directed graph of operations to transform input to + output. CWL is portable across a variety of computing + platforms. + + - {$include: intro.md} + + - | + ## Introduction to CWL Workflow standard v1.3.0-dev1 + + This specification represents the latest development version from the + CWL group. + + ## Changelog + See also the [CWL Command Line Tool Description, v1.3.0-dev1 changelog](CommandLineTool.html#Changelog). + For other changes since CWL v1.0, see the + [CWL Workflow Description, v1.1 changelog](https://www.commonwl.org/v1.1/Workflow.html#Changelog) + and + [CWL Workflow Description, v1.2 changelog](https://www.commonwl.org/v1.2/Workflow.html#Changelog) + + ## Purpose + + The Common Workflow Language Command Line Tool Description + express workflows for data-intensive science, such as + bioinformatics, physics, astronomy, geoscience, and machine + learning. This specification is intended to define a data and + execution model for Workflows that can be implemented on top of + a variety of computing platforms, ranging from an individual + workstation to cluster, grid, cloud, and high performance + computing systems. Details related to execution of these + workflow not laid out in this specification are open to + interpretation by the computing platform implementing this + specification. + + - {$include: concepts.md} + +- name: ExpressionToolOutputParameter + type: record + extends: OutputParameter + fields: + - name: type + type: + - CWLType + - OutputRecordSchema + - OutputEnumSchema + - OutputArraySchema + - string + - type: array + items: + - CWLType + - OutputRecordSchema + - OutputEnumSchema + - OutputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + Note that this field just acts as a hint, as the outputs of an + ExpressionTool process are always considered valid. + + +- name: WorkflowInputParameter + type: record + extends: InputParameter + docParent: "#Workflow" + fields: + - name: type + type: + - CWLType + - InputRecordSchema + - InputEnumSchema + - InputArraySchema + - string + - type: array + items: + - CWLType + - InputRecordSchema + - InputEnumSchema + - InputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + - name: inputBinding + type: InputBinding? + doc: | + Deprecated. Preserved for v1.0 backwards compatibility. Will be removed in + CWL v2.0. Use `WorkflowInputParameter.loadContents` instead. + jsonldPredicate: "cwl:inputBinding" + + +- type: record + name: ExpressionTool + extends: Process + specialize: + - specializeFrom: InputParameter + specializeTo: WorkflowInputParameter + - specializeFrom: OutputParameter + specializeTo: ExpressionToolOutputParameter + documentRoot: true + doc: | + An ExpressionTool is a type of Process object that can be run by itself + or as a Workflow step. It executes a pure Javascript expression that has + access to the same input parameters as a workflow. It is meant to be used + sparingly as a way to isolate complex Javascript expressions that need to + operate on input data and produce some result; perhaps just a + rearrangement of the inputs. No Docker software container is required + or allowed. + fields: + - name: class + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + type: + type: enum + name: ExpressionTool_class + symbols: + - cwl:ExpressionTool + - name: expression + type: Expression + doc: | + The expression to execute. The expression must return a plain + Javascript object which matches the output parameters of the + ExpressionTool. + +- name: LinkMergeMethod + type: enum + docParent: "#WorkflowStepInput" + doc: The input link merge method, described in [WorkflowStepInput](#WorkflowStepInput). + symbols: + - merge_nested + - merge_flattened + + +- name: PickValueMethod + type: enum + docParent: "#WorkflowStepInput" + doc: | + Picking non-null values among inbound data links, described in [WorkflowStepInput](#WorkflowStepInput). + symbols: + - first_non_null + - the_only_non_null + - all_non_null + + +- name: Sink + type: record + abstract: true + fields: + - name: linkMerge + type: LinkMergeMethod? + jsonldPredicate: "cwl:linkMerge" + default: merge_nested + doc: | + The method to use to merge multiple inbound links into a single array. + If not specified, the default method is "merge_nested". + - name: pickValue + type: [ "null", PickValueMethod ] + jsonldPredicate: "cwl:pickValue" + doc: | + The method to use to choose non-null elements among multiple sources. + + +- name: InputSink + type: record + extends: Sink + abstract: true + fields: + - name: source + doc: | + Specifies one or more workflow parameters that will provide input to + the underlying step parameter. + jsonldPredicate: + "_id": "cwl:source" + "_type": "@id" + refScope: 2 + type: + - string? + - string[]? + + +- name: OutputSink + type: record + extends: Sink + abstract: true + fields: + - name: outputSource + doc: | + Specifies one or more names of an output from a workflow step (in the form + `step_name/output_name` with a `/` separator`), or a workflow input name, + that supply their value(s) to the output parameter. + the output parameter. It is valid to reference workflow level inputs + here. + jsonldPredicate: + "_id": "cwl:outputSource" + "_type": "@id" + refScope: 1 + type: + - string? + - string[]? + + +- name: WorkflowOutputParameter + type: record + extends: [OutputParameter, OutputSink] + docParent: "#Workflow" + doc: | + Describe an output parameter of a workflow. The parameter must be + connected to one or more parameters defined in the workflow that + will provide the value of the output parameter. It is legal to + connect a WorkflowInputParameter to a WorkflowOutputParameter. + + See [WorkflowStepInput](#WorkflowStepInput) for discussion of + `linkMerge` and `pickValue`. + fields: + - name: type + type: + - CWLType + - OutputRecordSchema + - OutputEnumSchema + - OutputArraySchema + - string + - type: array + items: + - CWLType + - OutputRecordSchema + - OutputEnumSchema + - OutputArraySchema + - string + jsonldPredicate: + "_id": "sld:type" + "_type": "@vocab" + refScope: 2 + typeDSL: True + doc: | + Specify valid types of data that may be assigned to this parameter. + + +- type: record + name: WorkflowStepInput + extends: [Identified, InputSink, LoadContents, Labeled] + docParent: "#AbstractWorkflowStep" + doc: | + The input of a workflow step connects an upstream parameter (from the + workflow inputs, or the outputs of other workflows steps) with the input + parameters of the process specified by the `run` field. Only input parameters + declared by the target process will be passed through at runtime to the process + though additional parameters may be specified (for use within `valueFrom` + expressions for instance) - unconnected or unused parameters do not represent an + error condition. + + # Input object + + A WorkflowStepInput object must contain an `id` field in the form + `#fieldname` or `#prefix/fieldname`. When the `id` field contains a slash + `/` the field name consists of the characters following the final slash + (the prefix portion may contain one or more slashes to indicate scope). + This defines a field of the workflow step input object with the value of + the `source` parameter(s). + + # Merging multiple inbound data links + + To merge multiple inbound data links, + [MultipleInputFeatureRequirement](#MultipleInputFeatureRequirement) must be specified + in the workflow or workflow step requirements. + + If the sink parameter is an array, or named in a [workflow + scatter](#ScatterWorkflowStep) operation, there may be multiple inbound + data links listed in the `source` field. The values from the + input links are merged depending on the method specified in the + `linkMerge` field. If both `linkMerge` and `pickValue` are null + or not specified, and there is more than one element in the + `source` array, the default method is "merge_nested". + + If both `linkMerge` and `pickValue` are null or not specified, and + there is only a single element in the `source`, then the input + parameter takes the scalar value from the single input link (it is + *not* wrapped in a single-list). + + * **merge_nested** + + The input must be an array consisting of exactly one entry for each + input link. If "merge_nested" is specified with a single link, the value + from the link must be wrapped in a single-item list. + + * **merge_flattened** + + 1. The source and sink parameters must be compatible types, or the source + type must be compatible with single element from the "items" type of + the destination array parameter. + 2. Source parameters which are arrays are concatenated. + Source parameters which are single element types are appended as + single elements. + + # Picking non-null values among inbound data links + + If present, `pickValue` specifies how to pick non-null values among inbound data links. + + `pickValue` is evaluated + 1. Once all source values from upstream step or parameters are available. + 2. After `linkMerge`. + 3. Before `scatter` or `valueFrom`. + + This is specifically intended to be useful in combination with + [conditional execution](#AbstractWorkflowStep), where several upstream + steps may be connected to a single input (`source` is a list), and + skipped steps produce null values. + + Static type checkers should check for type consistency after inferring what the type + will be after `pickValue` is applied, just as they do currently for `linkMerge`. + + * **first_non_null** + + For the first level of a list input, pick the first non-null element. The result is a scalar. + It is an error if there is no non-null element. Examples: + * `[null, x, null, y] -> x` + * `[null, [null], null, y] -> [null]` + * `[null, null, null] -> Runtime Error` + + *Intended use case*: If-else pattern where the + value comes either from a conditional step or from a default or + fallback value. The conditional step(s) should be placed first in + the list. + + * **the_only_non_null** + + For the first level of a list input, pick the single non-null element. The result is a scalar. + It is an error if there is more than one non-null element. Examples: + + * `[null, x, null] -> x` + * `[null, x, null, y] -> Runtime Error` + * `[null, [null], null] -> [null]` + * `[null, null, null] -> Runtime Error` + + *Intended use case*: Switch type patterns where developer considers + more than one active code path as a workflow error + (possibly indicating an error in writing `when` condition expressions). + + * **all_non_null** + + For the first level of a list input, pick all non-null values. + The result is a list, which may be empty. Examples: + + * `[null, x, null] -> [x]` + * `[x, null, y] -> [x, y]` + * `[null, [x], [null]] -> [[x], [null]]` + * `[null, null, null] -> []` + + *Intended use case*: It is valid to have more than one source, but + sources are conditional, so null sources (from skipped steps) + should be filtered out. + + fields: + - name: default + type: CWLObjectType? + doc: | + The default value for this parameter to use if either there is no + `source` field, or the value produced by the `source` is `null`. The + default must be applied prior to scattering or evaluating `valueFrom`. + jsonldPredicate: + _id: "sld:default" + _container: "@list" + noLinkCheck: true + - name: valueFrom + type: + - "null" + - string + - Expression + jsonldPredicate: "cwl:valueFrom" + doc: | + To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must + be specified in the workflow or workflow step requirements. + + If `valueFrom` is a constant string value, use this as the value for + this input parameter. + + If `valueFrom` is a parameter reference or expression, it must be + evaluated to yield the actual value to be assigned to the input field. + + The `self` value in the parameter reference or expression must be + 1. `null` if there is no `source` field + 2. the value of the parameter(s) specified in the `source` field when this + workflow input parameter **is not** specified in this workflow step's `scatter` field. + 3. an element of the parameter specified in the `source` field when this workflow input + parameter **is** specified in this workflow step's `scatter` field. + + The value of `inputs` in the parameter reference or expression must be + the input object to the workflow step after assigning the `source` + values, applying `default`, and then scattering. The order of + evaluating `valueFrom` among step input parameters is undefined and the + result of evaluating `valueFrom` on a parameter must not be visible to + evaluation of `valueFrom` on other parameters. + + +- type: record + name: WorkflowStepOutput + docParent: "#AbstractWorkflowStep" + extends: Identified + doc: | + Associate an output parameter of the underlying process with a workflow + parameter. The workflow parameter (given in the `id` field) be may be used + as a `source` to connect with input parameters of other workflow steps, or + with an output parameter of the process. + + A unique identifier for this workflow output parameter. This is + the identifier to use in the `source` field of `WorkflowStepInput` + to connect the output value to downstream parameters. + + +- name: LoopInput + type: record + extends: [Identified, OutputSink] + fields: + - name: default + type: ["null", File, Directory, Any] + doc: | + The default value for this parameter to use if either there is no + `outputSource` field, or the value produced by the `source` is `null`. The + default must be applied prior to scattering or evaluating `valueFrom`. + jsonldPredicate: + _id: "sld:default" + _container: "@list" + noLinkCheck: true + - name: valueFrom + type: + - "null" + - string + - Expression + jsonldPredicate: "cwl:valueFrom" + doc: | + To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must + be specified in the workflow or workflow step requirements. + + If `valueFrom` is a constant string value, use this as the value for + this input parameter. + + If `valueFrom` is a parameter reference or expression, it must be + evaluated to yield the actual value to be assigned to the input field. + + The `self` value in the parameter reference or expression must be + `null` if there is no `source` field, or the value of the + parameter(s) specified in the `source` field. + + The value of `inputs` in the parameter reference or expression must be + the input object to the previous iteration of the workflow step (or the initial + inputs for the first iteration). + + +- name: ScatterMethod + type: enum + docParent: "#ScatterWorkflowStep" + doc: The scatter method, as described in [workflow step scatter](#ScatterWorkflowStep). + symbols: + - dotproduct + - nested_crossproduct + - flat_crossproduct + + +- name: LoopOutputMethod + type: enum + docParent: "#LoopWorkflowStep" + doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep). + symbols: + - last + - all + + +- name: AbstractWorkflowStep + type: record + extends: [Identified, Labeled, sld:Documented] + abstract: true + docParent: "#Workflow" + doc: | + A workflow step is an executable element of a workflow. It specifies the + underlying process implementation (such as `CommandLineTool` or another + `Workflow`) in the `run` field and connects the input and output parameters + of the underlying process to workflow parameters. + + # Conditional execution (Optional) + + Conditional execution makes execution of a step conditional on an + expression. A step that is not executed is "skipped". A skipped + step produces `null` for all output parameters. + + The `when` field controls conditional execution. This is an + expression that must be evaluated with `inputs` bound to the step + input object (or individual scatter job), and returns a boolean + value. It is an error if this expression returns a value other + than `true` or `false`. + + Conditional execution in CWL is an optional feature and is not required + to be implemented by all consumers of CWL documents. An implementation that + does not support conditional executions must return a fatal error when + attempting to execute a workflow that uses conditional constructs the + implementation does not support. + + # Subworkflows + + To specify a nested workflow as part of a workflow step, + [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) must be + specified in the workflow or workflow step requirements. + + It is a fatal error if a workflow directly or indirectly invokes itself as + a subworkflow (recursive workflows are not allowed). + + fields: + - name: in + type: WorkflowStepInput[] + jsonldPredicate: + _id: "cwl:in" + mapSubject: id + mapPredicate: source + doc: | + Defines the input parameters of the workflow step. The process is ready to + run when all required input parameters are associated with concrete + values. Input parameters include a schema for each parameter which is + used to validate the input object. It may also be used build a user + interface for constructing the input object. + - name: out + type: + - type: array + items: [string, WorkflowStepOutput] + jsonldPredicate: + _id: "cwl:out" + _type: "@id" + identity: true + doc: | + Defines the parameters representing the output of the process. May be + used to generate and/or validate the output object. + - name: requirements + type: ProcessRequirement[]? + jsonldPredicate: + _id: "cwl:requirements" + mapSubject: class + doc: | + Declares requirements that apply to either the runtime environment or the + workflow engine that must be met in order to execute this workflow step. If + an implementation cannot satisfy all requirements, or a requirement is + listed which is not recognized by the implementation, it is a fatal + error and the implementation must not attempt to run the process, + unless overridden at user option. + - name: hints + type: Any[]? + jsonldPredicate: + _id: "cwl:hints" + noLinkCheck: true + mapSubject: class + doc: | + Declares hints applying to either the runtime environment or the + workflow engine that may be helpful in executing this workflow step. It is + not an error if an implementation cannot satisfy all hints, however + the implementation may report a warning. + - name: run + type: [string, Process] + jsonldPredicate: + _id: "cwl:run" + _type: "@id" + subscope: run + doc: | + Specifies the process to run. If `run` is a string, it must be an absolute IRI + or a relative path from the primary document. + - name: when + type: + - "null" + - Expression + jsonldPredicate: "cwl:when" + doc: | + If defined, only run the step when the expression evaluates to + `true`. If `false` the step is skipped. A skipped step + produces a `null` on each output. + + +- name: WorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + + +- name: ScatterWorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + doc: | + To use scatter/gather, + [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified + in the workflow or workflow step requirements. + + A "scatter" operation specifies that the associated workflow step or + subworkflow should execute separately over a list of input elements. Each + job making up a scatter operation is independent and may be executed + concurrently. + + The `scatter` field specifies one or more input parameters which will be + scattered. An input parameter may be listed more than once. The declared + type of each input parameter implicitly becomes an array of items of the + input parameter type. If a parameter is listed more than once, it becomes + a nested array. As a result, upstream parameters which are connected to + scattered parameters must be arrays. + + All output parameter types are also implicitly wrapped in arrays. Each job + in the scatter results in an entry in the output array. + + If any scattered parameter runtime value is an empty array, all outputs are + set to empty arrays and no work is done for the step, according to + applicable scattering rules. + + If `scatter` declares more than one input parameter, `scatterMethod` + describes how to decompose the input into a discrete set of jobs. + + * **dotproduct** specifies that each of the input arrays are aligned and one + element taken from each array to construct each job. It is an error + if all input arrays are not the same length. + + * **nested_crossproduct** specifies the Cartesian product of the inputs, + producing a job for every combination of the scattered inputs. The + output must be nested arrays for each level of scattering, in the + order that the input arrays are listed in the `scatter` field. + + * **flat_crossproduct** specifies the Cartesian product of the inputs, + producing a job for every combination of the scattered inputs. The + output arrays must be flattened to a single level, but otherwise listed in the + order that the input arrays are listed in the `scatter` field. + + # Conditional execution (Optional) + + The condition is evaluated after `scatter`, using the input object + of each individual scatter job. This means over a set of scatter + jobs, some may be executed and some may be skipped. When the + results are gathered, skipped steps must be `null` in the output + arrays. + + fields: + - name: scatter + type: + - string? + - string[]? + jsonldPredicate: + "_id": "cwl:scatter" + "_type": "@id" + "_container": "@list" + refScope: 0 + - name: scatterMethod + doc: | + Required if `scatter` is an array of more than one element. + type: ScatterMethod? + jsonldPredicate: + "_id": "cwl:scatterMethod" + "_type": "@vocab" + + +- name: LoopWorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + doc: | + # Iterative execution (Optional) + + The `loop` field controls iterative execution. It defines the input + parameters of the loop iterations after the first one (inputs of the + first iteration are the step input parameters, as usual). If no + `loop` rule is specified for a given step `in` field, the initial + value is kept constant among all iterations. + + When a `loop` field is present, the `when` field is mandatory. It is + evaluated before each loop iteration and acts as a termination condition: + as soon as the `when` expression evaluates to `false`, the loop terminates + and the step outputs are propagated to the subsequent workflow steps. + + The `outputMethod` field describes how to deal with loop outputs after + termination: + + * **last** specifies that only the last computed element for each output + parameter should be propagated to the subsequent steps. This is the + default value. + + * **all** specifies that an array with all output values computed at the + end of each loop iteration should be propagated to the subsequent steps. + Elements in the array must be ordered according to the loop iterations + that produced them. + + Iterative execution in CWL is an optional feature and is not required + to be implemented by all consumers of CWL documents. An implementation that + does not support iterative executions must return a fatal error when + attempting to execute a workflow that uses iterative constructs the + implementation does not support. + + fields: + - name: loop + doc: | + Defines the input parameters of the loop iterations after the first one + (inputs of the first iteration are the step input parameters). If no + `loop` rule is specified for a given step `in` field, the initial value + is kept constant among all iterations. + type: LoopInput[]? + jsonldPredicate: + _id: "cwl:loop" + mapSubject: id + mapPredicate: outputSource + - name: outputMethod + doc: | + If not specified, the default method is "last". + type: LoopOutputMethod? + default: last + jsonldPredicate: + "_id": "cwl:outputMethod" + "_type": "@vocab" + - name: when + type: + - Expression + jsonldPredicate: "cwl:when" + doc: | + Only run the next iteration when the expression evaluates to `true`. + If the first iteration evaluates to `false` the step is skipped. + A skipped step produces a `null` on each output if the `outputMethod` + is set to `last`, and an empty array if the `outputMethod` is set to `all`. + + +- name: Workflow + type: record + extends: "#Process" + documentRoot: true + specialize: + - specializeFrom: InputParameter + specializeTo: WorkflowInputParameter + - specializeFrom: OutputParameter + specializeTo: WorkflowOutputParameter + doc: | + A workflow describes a set of **steps** and the **dependencies** between + those steps. When a step produces output that will be consumed by a + second step, the first step is a dependency of the second step. + + When there is a dependency, the workflow engine must execute the preceding + step and wait for it to successfully produce output before executing the + dependent step. If two steps are defined in the workflow graph that + are not directly or indirectly dependent, these steps are **independent**, + and may execute in any order or execute concurrently. A workflow is + complete when all steps have been executed. + + Dependencies between parameters are expressed using the `source` + field on [workflow step input parameters](#WorkflowStepInput) and + `outputSource` field on [workflow output + parameters](#WorkflowOutputParameter). + + The `source` field on each workflow step input parameter expresses + the data links that contribute to the value of the step input + parameter (the "sink"). A workflow step can only begin execution + when every data link connected to a step has been fulfilled. + + The `outputSource` field on each workflow step input parameter + expresses the data links that contribute to the value of the + workflow output parameter (the "sink"). Workflow execution cannot + complete successfully until every data link connected to an output + parameter has been fulfilled. + + ## Workflow success and failure + + A completed step must result in one of `success`, `temporaryFailure` or + `permanentFailure` states. An implementation may choose to retry a step + execution which resulted in `temporaryFailure`. An implementation may + choose to either continue running other steps of a workflow, or terminate + immediately upon `permanentFailure`. + + * If any step of a workflow execution results in `permanentFailure`, then + the workflow status is `permanentFailure`. + + * If one or more steps result in `temporaryFailure` and all other steps + complete `success` or are not executed, then the workflow status is + `temporaryFailure`. + + * If all workflow steps are executed and complete with `success`, then the + workflow status is `success`. + + # Extensions + + [ScatterFeatureRequirement](#ScatterFeatureRequirement) and + [SubworkflowFeatureRequirement](#SubworkflowFeatureRequirement) are + available as standard [extensions](#Extensions_and_Metadata) to core + workflow semantics. + + fields: + - name: "class" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + type: + type: enum + name: Workflow_class + symbols: + - cwl:Workflow + - name: steps + doc: | + The individual steps that make up the workflow. Each step is executed when all of its + input data links are fulfilled. An implementation may choose to execute + the steps in a different order than listed and/or execute steps + concurrently, provided that dependencies between steps are met. + type: + - type: array + items: AbstractWorkflowStep + jsonldPredicate: + mapSubject: id + + +- type: record + name: SubworkflowFeatureRequirement + extends: ProcessRequirement + doc: | + Indicates that the workflow platform must support nested workflows in + the `run` field of [AbstractWorkflowStep](#AbstractWorkflowStep). + fields: + - name: "class" + type: + type: enum + name: SubworkflowFeatureRequirement_class + symbols: + - cwl:SubworkflowFeatureRequirement + doc: "Always 'SubworkflowFeatureRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + +- name: ScatterFeatureRequirement + type: record + extends: ProcessRequirement + doc: | + Indicates that the workflow platform must support the `scatter` and + `scatterMethod` fields of [ScatterWorkflowStep](#ScatterWorkflowStep). + fields: + - name: "class" + type: + type: enum + name: ScatterFeatureRequirement_class + symbols: + - cwl:ScatterFeatureRequirement + doc: "Always 'ScatterFeatureRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + +- name: MultipleInputFeatureRequirement + type: record + extends: ProcessRequirement + doc: | + Indicates that the workflow platform must support multiple inbound data links + listed in the `source` field of [WorkflowStepInput](#WorkflowStepInput). + fields: + - name: "class" + type: + type: enum + name: MultipleInputFeatureRequirement_class + symbols: + - cwl:MultipleInputFeatureRequirement + doc: "Always 'MultipleInputFeatureRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + +- type: record + name: StepInputExpressionRequirement + extends: ProcessRequirement + doc: | + Indicate that the workflow platform must support the `valueFrom` field + of [WorkflowStepInput](#WorkflowStepInput). + fields: + - name: "class" + type: + type: enum + name: StepInputExpressionRequirement_class + symbols: + - cwl:StepInputExpressionRequirement + doc: "Always 'StepInputExpressionRequirement'" + jsonldPredicate: + "_id": "@type" + "_type": "@vocab" + +- {$import: Operation.yml} diff --git a/cwltool/schemas/v1.3.0-dev1/concepts.md b/cwltool/schemas/v1.3.0-dev1/concepts.md new file mode 100644 index 000000000..eda53d3f8 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/concepts.md @@ -0,0 +1,728 @@ +## References to other specifications + +**Javascript Object Notation (JSON)**: http://json.org + +**JSON Linked Data (JSON-LD)**: http://json-ld.org + +**YAML**: http://yaml.org + +**Avro**: https://avro.apache.org/docs/1.8.1/spec.html + +**Internationalized Resource Identifiers (IRIs)**: +https://tools.ietf.org/html/rfc3987 + +**Portable Operating System Interface (POSIX.1-2008)**: http://pubs.opengroup.org/onlinepubs/9699919799/ + +**Resource Description Framework (RDF)**: http://www.w3.org/RDF/ + +**XDG Base Directory Specification**: https://specifications.freedesktop.org/basedir-spec/basedir-spec-0.6.html + + +## Scope + +This document describes CWL syntax, execution, and object model. It +is not intended to document a CWL specific implementation, however it may +serve as a reference for the behavior of conforming implementations. + +## Terminology + +The terminology used to describe CWL documents is defined in the +Concepts section of the specification. The terms defined in the +following list are used in building those definitions and in describing the +actions of a CWL implementation: + +**may**: Conforming CWL documents and CWL implementations are permitted but +not required to behave as described. + +**must**: Conforming CWL documents and CWL implementations are required to behave +as described; otherwise they are in error. + +**error**: A violation of the rules of this specification; results are +undefined. Conforming implementations may detect and report an error and may +recover from it. + +**fatal error**: A violation of the rules of this specification; results are +undefined. Conforming implementations must not continue to execute the current +process and may report an error. + +**at user option**: Conforming software may or must (depending on the modal verb in +the sentence) behave as described; if it does, it must provide users a means to +enable or disable the behavior described. + +**deprecated**: Conforming software may implement a behavior for backwards +compatibility. Portable CWL documents should not rely on deprecated behavior. +Behavior marked as deprecated may be removed entirely from future revisions of +the CWL specification. + +## Glossary + +**Opaque strings**: Opaque strings +(or opaque identifiers, opaque values) are nonsensical values that are +swapped out with a real value later in the evaluation process. Workflow +and tool expressions **should not** rely on it nor try to parse it. + +# Data model + +## Data concepts + +An **object** is a data structure equivalent to the "object" type in JSON, +consisting of an unordered set of name/value pairs (referred to here as +**fields**) and where the name is a string and the value is a string, number, +boolean, array, or object. + +A **document** is a file containing a serialized object, or an array of objects. + +A **process** is a basic unit of computation which accepts input data, +performs some computation, and produces output data. Examples include +CommandLineTools, Workflows, and ExpressionTools. + +An **input object** is an object describing the inputs to an +invocation of a process. The fields of the input object are referred +to as "input parameters". + +An **output object** is an object describing the output resulting from +an invocation of a process. The fields of the output object are +referred to as "output parameters". + +An **input schema** describes the valid format (required fields, data types) +for an input object. + +An **output schema** describes the valid format for an output object. + +**Metadata** is information about workflows, tools, or input items. + +## Syntax + +CWL documents must consist of an object or array of objects represented using +JSON or YAML syntax. Upon loading, a CWL implementation must apply the +preprocessing steps described in the +[Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html). +An implementation may formally validate the structure of a CWL document using +SALAD schemas located at https://github.com/common-workflow-language/cwl-v1.2/ + +The official IANA media-type for CWL documents is [`application/cwl`](https://www.iana.org/assignments/media-types/application/cwl) +for either JSON or YAML format. For JSON formatted CWL documents, +[`application/cwl+json`](https://www.iana.org/assignments/media-types/application/cwl+json) +can be used. For specifying a YAML formatted CWL document, one can use +`application/cwl+yaml` but that is not an official IANA media-type yet; as of +2023-07-23 the `+yaml` suffix has yet to be approved. + +CWL documents commonly reference other CWL documents. Each document +must declare the `cwlVersion` of that document. Implementations must +validate against the document's declared version. Implementations +should allow workflows to reference documents of both newer and older +CWL versions (up to the highest version of CWL supported by that +implementation). Where the runtime environment or runtime behavior has +changed between versions, for that portion of the execution an +implementation must provide runtime environment and behavior consistent +with the document's declared version. An implementation must not +expose a newer feature when executing a document that specifies an +older version that does not include that feature. + +### map + +Note: This section is non-normative. +> type: array<ComplexType> | +> map<`key_field`, ComplexType> + +The above syntax in the CWL specifications means there are two or more ways to write the given value. + +Option one is an array and is the most verbose option. + +Option one generic example: +``` +some_cwl_field: + - key_field: a_complex_type1 + field2: foo + field3: bar + - key_field: a_complex_type2 + field2: foo2 + field3: bar2 + - key_field: a_complex_type3 +``` + +Option one specific example using [Workflow](Workflow.html#Workflow).[inputs](Workflow.html#WorkflowInputParameter): +> array<InputParameter> | +> map<`id`, `type` | InputParameter> + + +``` +inputs: + - id: workflow_input01 + type: string + - id: workflow_input02 + type: File + format: http://edamontology.org/format_2572 +``` + +Option two is enabled by the `map<…>` syntax. Instead of an array of entries we +use a mapping, where one field of the `ComplexType` (here named `key_field`) +becomes the key in the map, and its value is the rest of the `ComplexType` +without the key field. If all of the other fields of the `ComplexType` are +optional and unneeded, then we can indicate this with an empty mapping as the +value: `a_complex_type3: {}` + +Option two generic example: +``` +some_cwl_field: + a_complex_type1: # this was the "key_field" from above + field2: foo + field3: bar + a_complex_type2: + field2: foo2 + field3: bar2 + a_complex_type3: {} # we accept the default values for "field2" and "field3" +``` + +Option two specific example using [Workflow](Workflow.html#Workflow).[inputs](Workflow.html#WorkflowInputParameter): +> array<InputParameter> | +> map<`id`, `type` | InputParameter> + + +``` +inputs: + workflow_input01: + type: string + workflow_input02: + type: File + format: http://edamontology.org/format_2572 +``` + +Option two specific example using [SoftwareRequirement](#SoftwareRequirement).[packages](#SoftwarePackage): +> array<SoftwarePackage> | +> map<`package`, `specs` | SoftwarePackage> + + +``` +hints: + SoftwareRequirement: + packages: + sourmash: + specs: [ https://doi.org/10.21105/joss.00027 ] + screed: + version: [ "1.0" ] + python: {} +``` + +Sometimes we have a third and even more compact option denoted like this: +> type: array<ComplexType> | +> map<`key_field`, `field2` | ComplexType> + +For this example, if we only need the `key_field` and `field2` when specifying +our `ComplexType`s (because the other fields are optional and we are fine with +their default values) then we can abbreviate. + +Option three generic example: +``` +some_cwl_field: + a_complex_type1: foo # we accept the default value for field3 + a_complex_type2: foo2 # we accept the default value for field3 + a_complex_type3: {} # we accept the default values for "field2" and "field3" +``` + +Option three specific example using [Workflow](Workflow.html#Workflow).[inputs](Workflow.html#WorkflowInputParameter): +> array<InputParameter> | +> map<`id`, `type` | InputParameter> + + +``` +inputs: + workflow_input01: string + workflow_input02: File # we accept the default of no File format +``` + +Option three specific example using [SoftwareRequirement](#SoftwareRequirement).[packages](#SoftwarePackage): +> array<SoftwarePackage> | +> map<`package`, `specs` | SoftwarePackage> + + +``` +hints: + SoftwareRequirement: + packages: + sourmash: [ https://doi.org/10.21105/joss.00027 ] + python: {} +``` + + +What if some entries we want to mix the option 2 and 3? You can! + +Mixed option 2 and 3 generic example: +``` +some_cwl_field: + my_complex_type1: foo # we accept the default value for field3 + my_complex_type2: + field2: foo2 + field3: bar2 # we did not accept the default value for field3 + # so we had to use the slightly expanded syntax + my_complex_type3: {} # as before, we accept the default values for both + # "field2" and "field3" +``` + +Mixed option 2 and 3 specific example using [Workflow](Workflow.html#Workflow).[inputs](Workflow.html#WorkflowInputParameter): +> array<InputParameter> | +> map<`id`, `type` | InputParameter> + + +``` +inputs: + workflow_input01: string + workflow_input02: # we use the longer way + type: File # because we want to specify the "format" too + format: http://edamontology.org/format_2572 +``` + +Mixed option 2 and 3 specific example using [SoftwareRequirement](#SoftwareRequirement).[packages](#SoftwarePackage): +> array<SoftwarePackage> | +> map<`package`, `specs` | SoftwarePackage> + + +``` +hints: + SoftwareRequirement: + packages: + sourmash: [ https://doi.org/10.21105/joss.00027 ] + screed: + specs: [ https://github.com/dib-lab/screed ] + version: [ "1.0" ] + python: {} +``` + +Note: The `map<…>` (compact) versions are optional for users, the verbose option #1 is +always allowed, but for presentation reasons option 3 and 2 may be preferred +by human readers. Consumers of CWL must support all three options. + +The normative explanation for these variations, aimed at implementers, is in the +[Schema Salad specification](SchemaSalad.html#Identifier_maps). + +## Identifiers + +If an object contains an `id` field, that is used to uniquely identify the +object in that document. The value of the `id` field must be unique over the +entire document. Identifiers may be resolved relative to either the document +base and/or other identifiers following the rules are described in the +[Schema Salad specification](SchemaSalad.html#Identifier_resolution). + +An implementation may choose to only honor references to object types for +which the `id` field is explicitly listed in this specification. + +## Document preprocessing + +An implementation must resolve [$import](SchemaSalad.html#Import) and +[$include](SchemaSalad.html#Include) directives as described in the +[Schema Salad specification](SchemaSalad.html#Document_preprocessing). + +Another transformation defined in Schema salad is simplification of data type definitions. +Type `` ending with `?` should be transformed to `[, "null"]`. +Type `` ending with `[]` should be transformed to `{"type": "array", "items": }`. +This simplification should be applied recursively. + +## Extensions and metadata + +Input metadata (for example, a sample identifier) may be represented within +a tool or workflow using input parameters which are explicitly propagated to +output. Future versions of this specification may define additional facilities +for working with input/output metadata. + +Implementation extensions not required for correct execution (for example, +fields related to GUI presentation) and metadata about the tool or workflow +itself (for example, authorship for use in citations) may be provided as +additional fields on any object. Such extensions fields must use a namespace +prefix listed in the `$namespaces` section of the document as described in the +[Schema Salad specification](SchemaSalad.html#Explicit_context). + +It is recommended that concepts from schema.org are used whenever possible. +For the `$schemas` field we recommend their RDF encoding: https://schema.org/version/latest/schemaorg-current-https.rdf + +Implementation extensions which modify execution semantics must be [listed in +the `requirements` field](#Requirements_and_hints). + +## Packed documents + +A "packed" CWL document is one that contains multiple process objects. +This makes it possible to store and transmit a Workflow together with +the processes of each of its steps in a single file. + +There are two methods to create packed documents: embedding and $graph. +These can be both appear in the same document. + +"Embedding" is where the entire process object is copied into the +`run` field of a workflow step. If the step process is a subworkflow, +it can be processed recursively to embed the processes of the +subworkflow steps, and so on. Embedded process objects may optionally +include `id` fields. + +A "$graph" document does not have a process object at the root. +Instead, there is a [`$graph`](SchemaSalad.html#Document_graph) field +which consists of a list of process objects. Each process object must +have an `id` field. Workflow `run` fields cross-reference other +processes in the document `$graph` using the `id` of the process +object. + +All process objects in a packed document must validate and execute as +the `cwlVersion` appearing the top level. A `cwlVersion` field +appearing anywhere other than the top level must be ignored. + +When executing a packed document, the reference to the document may +include a fragment identifier. If present, the fragment identifier +specifies the `id` of the process to execute. + +If the reference to the packed document does not include a fragment +identifier, the runner must choose the top-level process object as the +entry point. If there is no top-level process object (as in the case +of `$graph`) then the runner must choose the process object with an id +of `#main`. If there is no `#main` object, the runner must return an +error. + +# Execution model + +## Execution concepts + +A **parameter** is a named symbolic input or output of process, with +an associated datatype or schema. During execution, values are +assigned to parameters to make the input object or output object used +for concrete process invocation. + +A **CommandLineTool** is a process characterized by the execution of a +standalone, non-interactive program which is invoked on some input, +produces output, and then terminates. + +A **workflow** is a process characterized by multiple subprocess steps, +where step outputs are connected to the inputs of downstream steps to +form a directed acyclic graph, and independent steps may run concurrently. + +A **runtime environment** is the actual hardware and software environment when +executing a command line tool. It includes, but is not limited to, the +hardware architecture, hardware resources, operating system, software runtime +(if applicable, such as the specific Python interpreter or the specific Java +virtual machine), libraries, modules, packages, utilities, and data files +required to run the tool. + +A **workflow platform** is a specific hardware and software implementation +capable of interpreting CWL documents and executing the processes specified by +the document. The responsibilities of the workflow platform may include +scheduling process invocation, setting up the necessary runtime environment, +making input data available, invoking the tool process, and collecting output. + +A **data link** is a connection from a "Source" parameter to a "Sink" +parameter. A data link expresses that when a value becomes available +for the source parameter, that value should be copied to the "sink" +parameter. Reflecting the direction of data flow, a data link is +described as "outgoing" from the source and "inbound" to the sink. + +A workflow platform may choose to only implement the Command Line Tool +Description part of the CWL specification. + +It is intended that the workflow platform has broad leeway outside of this +specification to optimize use of computing resources and enforce policies +not covered by this specification. Some areas that are currently out of +scope for CWL specification but may be handled by a specific workflow +platform include: + +* Data security and permissions +* Scheduling tool invocations on remote cluster or cloud compute nodes. +* Using virtual machines or operating system containers to manage the runtime +(except as described in [DockerRequirement](CommandLineTool.html#DockerRequirement)). +* Using remote or distributed file systems to manage input and output files. +* Transforming file paths. +* Pausing, resuming or checkpointing processes or workflows. + +Conforming CWL processes must not assume anything about the runtime +environment or workflow platform unless explicitly declared though the use +of [process requirements](#Requirements_and_hints). + +## Generic execution process + +The generic execution sequence of a CWL process (including workflows +and command line tools) is as follows. Processes are +modeled as functions that consume an input object and produce an +output object. + +1. Load input object. +1. Load, process and validate a CWL document, yielding one or more process objects. +The [`$namespaces`](SchemaSalad.html#Explicit_context) present in the CWL document +are also used when validating and processing the input object. +1. If there are multiple process objects (due to [`$graph`](SchemaSalad.html#Document_graph)) +and which process object to start with is not specified in the input object (via +a [`cwl:tool`](#Executing_CWL_documents_as_scripts) entry) or by any other means +(like a URL fragment) then choose the process with the `id` of "#main" or "main". +1. Validate the input object against the `inputs` schema for the process. +1. Validate process requirements are met. +1. Perform any further setup required by the specific process type. +1. Execute the process. +1. Capture results of process execution into the output object. +1. Validate the output object against the `outputs` schema for the process (with the exception of ExpressionTool outputs, which are always considered valid). +1. Report the output object to the process caller. + +## Requirements and hints + +A **process requirement** modifies the semantics or runtime +environment of a process. If an implementation cannot satisfy all +requirements, or a requirement is listed which is not recognized by the +implementation, it is a fatal error and the implementation must not attempt +to run the process, unless overridden at user option. + +A **hint** is similar to a requirement; however, it is not an error if an +implementation cannot satisfy all hints. The implementation may report a +warning if a hint cannot be satisfied. + +Optionally, implementations may allow requirements to be specified in the input +object document as an array of requirements under the field name +`cwl:requirements`. If implementations allow this, then such requirements +should be combined with any requirements present in the corresponding Process +as if they were specified there. + +Requirements specified in a parent Workflow are inherited by step processes +if they are valid for that step. If the substep is a CommandLineTool +only the `InlineJavascriptRequirement`, `SchemaDefRequirement`, `DockerRequirement`, +`SoftwareRequirement`, `InitialWorkDirRequirement`, `EnvVarRequirement`, +`ShellCommandRequirement`, `ResourceRequirement`, `LoadListingRequirement`, +`WorkReuse`, `NetworkAccess`, `InplaceUpdateRequirement`, `ToolTimeLimit` are valid. + +*As good practice, it is best to have process requirements be self-contained, +such that each process can run successfully by itself.* + +If the same process requirement appears at different levels of the +workflow, the most specific instance of the requirement is used, that is, +an entry in `requirements` on a process implementation such as +CommandLineTool will take precedence over an entry in `requirements` +specified in a workflow step, and an entry in `requirements` on a workflow +step takes precedence over the workflow. Entries in `hints` are resolved +the same way. + +Requirements override hints. If a process implementation provides a +process requirement in `hints` which is also provided in `requirements` by +an enclosing workflow or workflow step, the enclosing `requirements` takes +precedence. + +## Parameter references + +Parameter references are denoted by the syntax `$(...)` and may be used in any +field permitting the pseudo-type `Expression`, as specified by this document. +Conforming implementations must support parameter references. Parameter +references use the following subset of +[Javascript/ECMAScript 5.1](http://www.ecma-international.org/ecma-262/5.1/) +syntax, but they are designed to not require a Javascript engine for evaluation. + +In the following [BNF grammar](https://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form), +character classes and grammar rules are denoted in `{}`, `-` denotes +exclusion from a character class, `(())` denotes grouping, `|` denotes +alternates, trailing `*` denotes zero or more repeats, `+` denotes one +or more repeats, and all other characters are literal values. + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
symbol::={Unicode alphanumeric}+
singleq::=[' (( {character - { | \ ' \} } ))* ']
doubleq::=[" (( {character - { | \ " \} } ))* "]
index::=[ {decimal digit}+ ]
segment::=. {symbol} | {singleq} | {doubleq} | {index}
parameter reference::=( {symbol} {segment}*)
+
+ +Use the following algorithm to resolve a parameter reference: + + 1. Match the leading symbol as the key + 2. If the key is the special value 'null' then the + value of the parameter reference is 'null'. If the key is 'null' it must be the only symbol in the parameter reference. + 3. Look up the key in the parameter context (described below) to get the current value. + It is an error if the key is not found in the parameter context. + 4. If there are no subsequent segments, terminate and return current value + 5. Else, match the next segment + 6. Extract the symbol, string, or index from the segment as the key + 7. Look up the key in current value and assign as new current value. + 1. If the key is a symbol or string, the current value must be an object. + 2. If the key is an index, the current value must be an array or string. + 3. If the next key is the last key and it has the special value 'length' and + the current value is an array, the value of the parameter reference is the + length of the array. If the value 'length' is encountered in other contexts, normal + evaluation rules apply. + 4. It is an error if the key does not match the required type, or the key is not found or out + of range. + 8. Repeat steps 3-8 + +The root namespace is the parameter context. The following parameters must +be provided: + + * `inputs`: The input object to the current Process. + * `self`: A context-specific value. The contextual values for 'self' are + documented for specific fields elsewhere in this specification. If + a contextual value of 'self' is not documented for a field, it + must be 'null'. + * `runtime`: An object containing configuration details. Specific to the + process type. An implementation may provide + [opaque strings](#opaque-strings) for any or all fields of `runtime`. + These must be filled in by the platform after processing the Tool but + before actual execution. Parameter references and expressions may only + use the literal string value of the field and must not perform computation + on the contents, except where noted otherwise. + +If the value of a field has no leading or trailing non-whitespace +characters around a parameter reference, the effective value of the field +becomes the value of the referenced parameter, preserving the return type. + +### String interpolation + +If the value of a field has non-whitespace leading or trailing characters +around a parameter reference, it is subject to string interpolation. The +effective value of the field is a string containing the leading characters, +followed by the string value of the parameter reference, followed by the +trailing characters. The string value of the parameter reference is its +textual JSON representation with the following rules: + + * Strings are replaced the literal text of the string, any escaped + characters replaced by the literal characters they represent, and + there are no leading or trailing quotes. + * Objects entries are sorted by key + +Multiple parameter references may appear in a single field. This case +must be treated as a string interpolation. After interpolating the first +parameter reference, interpolation must be recursively applied to the +trailing characters to yield the final string value. + +When text embedded in a CWL file represents code for another +programming language, the use of `$(...)` (and `${...}` in the case of +expressions) may conflict with the syntax of that language. For +example, when writing shell scripts, `$(...)` is used to execute a +command in a subshell and replace a portion of the command line with +the standard output of that command. + +The following escaping rules apply. The scanner makes a single pass +from start to end with 3-character lookahead. After performing a +replacement scanning resumes at the next character following the +replaced substring. + + 1. The substrings `\$(` and `\${` are replaced by `$(` and `${` + respectively. No parameter or expression evaluation + interpolation occurs. + 2. A double backslash `\\` is replaced by a single backslash `\`. + 3. A substring starting with a backslash that does not match one of + the previous rules is left unchanged. + +## Expressions (Optional) + +An expression is a fragment of [Javascript/ECMAScript +5.1](http://www.ecma-international.org/ecma-262/5.1/) code evaluated by the +workflow platform to affect the inputs, outputs, or +behavior of a process. In the generic execution sequence, expressions may +be evaluated during step 5 (process setup), step 6 (execute process), +and/or step 7 (capture output). Expressions are distinct from regular +processes in that they are intended to modify the behavior of the workflow +itself rather than perform the primary work of the workflow. + +Expressions in CWL are an optional feature and are not required to be +implemented by all consumers of CWL documents. They should be used sparingly, +when there is no other way to achieve the desired outcome. Excessive use of +expressions may be a signal that other refactoring of the tools or workflows +would benefit the author, runtime, and users of the CWL document in question. + +To declare the use of expressions, the document must include the process +requirement `InlineJavascriptRequirement`. Expressions may be used in any +field permitting the pseudo-type `Expression`, as specified by this +document. + +Expressions are denoted by the syntax `$(...)` or `${...}`. + +A code fragment wrapped in the `$(...)` syntax must be evaluated as a +[ECMAScript expression](http://www.ecma-international.org/ecma-262/5.1/#sec-11). + +A code fragment wrapped in the `${...}` syntax must be evaluated as a +[ECMAScript function body](http://www.ecma-international.org/ecma-262/5.1/#sec-13) +for an anonymous, zero-argument function. This means the code will be +evaluated as `(function() { ... })()`. + +Expressions must return a valid JSON data type: one of null, string, number, +boolean, array, object. Other return values must result in a +`permanentFailure`. Implementations must permit any syntactically valid +Javascript and account for nesting of parenthesis or braces and that strings +that may contain parenthesis or braces when scanning for expressions. + +The runtime must include any code defined in the ["expressionLib" field of +InlineJavascriptRequirement](#InlineJavascriptRequirement) prior to +executing the actual expression. + +Before executing the expression, the runtime must initialize as global +variables the fields of the parameter context described above. + +The effective value of the field after expression evaluation follows the +same rules as parameter references discussed above. Multiple expressions +may appear in a single field. + +Expressions must be evaluated in an isolated context (a "sandbox") which +permits no side effects to leak outside the context. Expressions also must +be evaluated in [Javascript strict mode](http://www.ecma-international.org/ecma-262/5.1/#sec-4.2.2). + +The order in which expressions are evaluated is undefined except where +otherwise noted in this document. + +An implementation may choose to implement parameter references by +evaluating as a Javascript expression. The results of evaluating +parameter references must be identical whether implemented by Javascript +evaluation or some other means. + +Implementations may apply other limits, such as process isolation, timeouts, +and operating system containers/jails to minimize the security risks associated +with running untrusted code embedded in a CWL document. + +Javascript exceptions thrown from a CWL expression must result in a +`permanentFailure` of the CWL process. + +## Executing CWL documents as scripts + +By convention, a CWL document may begin with `#!/usr/bin/env cwl-runner` +and be marked as executable (the POSIX "+x" permission bits) to enable it +to be executed directly. A workflow platform may support this mode of +operation; if so, it must provide `cwl-runner` as an alias for the +platform's CWL implementation. + +A CWL input object document may similarly begin with `#!/usr/bin/env +cwl-runner` and be marked as executable. In this case, the input object +must include the field `cwl:tool` supplying an IRI to the default CWL +document that should be executed using the fields of the input object as +input parameters. + +The `cwl-runner` interface is required for conformance testing and is +documented in [cwl-runner.cwl](cwl-runner.cwl). + +## Discovering CWL documents on a local filesystem + +To discover CWL documents look in the following locations: + +For each value in the `XDG_DATA_DIRS` environment variable (which is a `:` colon +separated list), check the `./commonwl` subdirectory. If `XDG_DATA_DIRS` is +unset or empty, then check using the default value for `XDG_DATA_DIRS`: +`/usr/local/share/:/usr/share/` (That is to say, check `/usr/share/commonwl/` +and `/usr/local/share/commonwl/`) + +Then check `$XDG_DATA_HOME/commonwl/`. + +If the `XDG_DATA_HOME` environment variable is unset, its default value is +`$HOME/.local/share` (That is to say, check `$HOME/.local/share/commonwl`) + +`$XDG_DATA_HOME` and `$XDG_DATA_DIRS` are from the [XDG Base Directory +Specification](http://standards.freedesktop.org/basedir-spec/basedir-spec-0.6.html) diff --git a/cwltool/schemas/v1.3.0-dev1/conformance_tests.yaml b/cwltool/schemas/v1.3.0-dev1/conformance_tests.yaml new file mode 100644 index 000000000..ee8c0a6cf --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/conformance_tests.yaml @@ -0,0 +1,3529 @@ +# every "id" must be unique and have no spaces +# every "doc" must be unique +# "id" and "doc" should be meaningful + +- job: tests/bwa-mem-job.json + tool: tests/bwa-mem-tool.cwl + output: + args: [bwa, mem, -t, '2', -I, '1,2,3,4', -m, '3', + chr20.fa, + example_human_Illumina.pe_1.fastq, + example_human_Illumina.pe_2.fastq] + id: cl_basic_generation + doc: General test of command line generation + tags: [ required, command_line_tool ] + +- output: + args: [bwa, mem, chr20.fa, + "-XXX", + "-YYY", example_human_Illumina.pe_1.fastq, + "-YYY", example_human_Illumina.pe_2.fastq] + job: tests/bwa-mem-job.json + tool: tests/binding-test.cwl + id: nested_prefixes_arrays + doc: Test nested prefixes with arrays + tags: [ required, command_line_tool ] + +- output: + args: [tmap, mapall, stage1, map1, --min-seq-length, '20', map2, --min-seq-length, + '20', stage2, map1, --max-seq-length, '20', --min-seq-length, '10', --seed-length, + '16', map2, --max-seed-hits, '-1', --max-seq-length, '20', --min-seq-length, '10'] + job: tests/tmap-job.json + tool: tests/tmap-tool.cwl + id: nested_cl_bindings + doc: Test nested command line bindings + tags: [ schema_def, command_line_tool ] + +- output: + args: [cat, hello.txt] + job: tests/cat-job.json + tool: tests/cat1-testcli.cwl + id: cl_optional_inputs_missing + doc: Test command line with optional input (missing) + tags: [ required, command_line_tool ] + +- output: + args: [cat, -n, hello.txt] + job: tests/cat-n-job.json + tool: tests/cat1-testcli.cwl + id: cl_optional_bindings_provided + doc: Test command line with optional input (provided) + tags: [ required, command_line_tool ] + +- output: + "foo": { + "checksum": "sha1$63da67422622fbf9251a046d7a34b7ea0fd4fead", + "class": "File", + "location": "foo.txt", + "size": 22 + } + job: tests/cat-job.json + tool: tests/template-tool.cwl + id: initworkdir_expreng_requirements + doc: Test InitialWorkDirRequirement ExpressionEngineRequirement.engineConfig feature + tags: [ initial_work_dir, inline_javascript, command_line_tool ] + +- job: tests/cat-job.json + output: + output_file: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + location: output.txt + size: 13 + tool: tests/cat3-tool.cwl + id: stdout_redirect_docker + doc: Test command execution in Docker with stdout redirection + tags: [ docker, command_line_tool ] + +- job: tests/cat-job.json + tool: tests/cat3-tool-shortcut.cwl + output: + output_file: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + location: Any + size: 13 + id: stdout_redirect_shortcut_docker + doc: Test command execution in Docker with shortcut stdout redirection + tags: [ docker, command_line_tool ] + +- job: tests/cat-job.json + output: + output_file: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + location: cat-out + size: 13 + tool: tests/cat3-tool-mediumcut.cwl + id: stdout_redirect_mediumcut_docker + doc: Test command execution in Docker with mediumcut stdout redirection + tags: [ docker, command_line_tool ] + +- job: tests/empty.json + tool: tests/stderr.cwl + id: stderr_redirect + doc: Test command line with stderr redirection + output: + output_file: + class: File + checksum: sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 + size: 4 + location: error.txt + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + tool: tests/stderr-shortcut.cwl + id: stderr_redirect_shortcut + doc: Test command line with stderr redirection, brief syntax + output: + output_file: + class: File + checksum: sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 + size: 4 + location: Any + tags: [ shell_command, command_line_tool ] + +- output: + output_file: + class: File + size: 4 + checksum: sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15 + location: std.err + job: tests/empty.json + tool: tests/stderr-mediumcut.cwl + id: stderr_redirect_mediumcut + doc: Test command line with stderr redirection, named brief syntax + tags: [ shell_command, command_line_tool ] + +- job: tests/cat-job.json + output: + output_txt: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + location: output.txt + size: 13 + tool: tests/cat4-tool.cwl + id: stdinout_redirect_docker + doc: Test command execution in Docker with stdin and stdout redirection + tags: [ required, command_line_tool ] + +- job: tests/empty.json + tool: tests/null-expression1-tool.cwl + output: + output: 1 + id: expression_any + doc: Test default usage of Any in expressions. + tags: [ inline_javascript, expression_tool ] + +- job: tests/null-expression1-job.json + tool: tests/null-expression1-tool.cwl + output: + output: 1 + id: expression_any_null + doc: Test explicitly passing null to Any type inputs with default values. + tags: [ inline_javascript, expression_tool ] + +- job: tests/null-expression2-job.json + tool: tests/null-expression1-tool.cwl + output: + output: 2 + id: expression_any_string + doc: Testing the string 'null' does not trip up an Any with a default value. + tags: [ inline_javascript, expression_tool ] + +- job: tests/empty.json + tool: tests/null-expression2-tool.cwl + should_fail: true + id: expression_any_nodefaultany + doc: Test Any without defaults cannot be unspecified. + tags: [ inline_javascript, expression_tool ] + +- job: tests/null-expression1-job.json + tool: tests/null-expression2-tool.cwl + should_fail: true + id: expression_any_null_nodefaultany + doc: Test explicitly passing null to Any type without a default value. + tags: [ inline_javascript, expression_tool ] + +- job: tests/null-expression2-job.json + tool: tests/null-expression2-tool.cwl + output: + output: 2 + id: expression_any_nullstring_nodefaultany + doc: Testing the string 'null' does not trip up an Any without a default value. + tags: [ inline_javascript, expression_tool ] + +- job: tests/any-type-job.json + tool: tests/any-type-compat.cwl + output: + output1: ["hello", "world"] + output2: ["foo", "bar"] + output3: hello + id: any_outputSource_compatibility + doc: Testing Any type compatibility in outputSource + tags: [ required, workflow ] + +- job: tests/cat-job.json + output: + output: + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + class: File + location: output + size: 13 + tool: tests/cat-tool.cwl + id: stdinout_redirect + doc: Test command execution in with stdin and stdout redirection + tags: [ required, command_line_tool ] + +- job: tests/parseInt-job.json + output: {output: 42} + tool: tests/parseInt-tool.cwl + id: expression_parseint + doc: Test ExpressionTool with Javascript engine + tags: [ inline_javascript, expression_tool ] + +- job: tests/wc-job.json + output: {output: 16} + tool: tests/wc2-tool.cwl + id: expression_outputEval + doc: Test outputEval to transform output + tags: [ inline_javascript, command_line_tool ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines1-wf.cwl + id: wf_wc_parseInt + doc: Test two step workflow with imported tools + tags: [ inline_javascript, workflow ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines2-wf.cwl + id: wf_wc_expressiontool + doc: Test two step workflow with inline tools + tags: [ inline_javascript, workflow ] + +- job: tests/count-lines3-job.json + output: + count_output: [16, 1] + tool: tests/count-lines3-wf.cwl + id: wf_wc_scatter + doc: Test single step workflow with Scatter step + tags: [ scatter, inline_javascript, workflow ] + +- job: tests/count-lines4-job.json + output: + count_output: [16, 1] + tool: tests/count-lines4-wf.cwl + id: wf_wc_scatter_multiple_merge + doc: >- + Test single step workflow with Scatter step and two data links connected to + same input, default merge behavior + tags: [ scatter, multiple_input, inline_javascript, workflow ] + +- job: tests/count-lines6-job.json + output: + count_output: [32, 2] + tool: tests/count-lines6-wf.cwl + id: wf_wc_scatter_multiple_nested + doc: >- + Test single step workflow with Scatter step and two data links connected to + same input, nested merge behavior + tags: [ scatter, multiple_input, inline_javascript, workflow ] + +- job: tests/count-lines6-job.json + output: + count_output: 34 + tool: tests/count-lines7-wf.cwl + id: wf_wc_scatter_multiple_flattened + doc: >- + Test single step workflow with Scatter step and two data links connected to + same input, flattened merge behavior + tags: [ multiple_input, inline_javascript, workflow ] + +- job: tests/count-lines4-job.json + output: + count_output: 16 + tool: tests/count-lines13-wf.cwl + id: wf_wc_nomultiple + doc: >- + Test when step source is a single-item list and there is no + linkMerge, then it not wrapped in a list, and that + MultipleInputFeatureRequirement is not required. + tags: [ inline_javascript, workflow ] + +- job: tests/count-lines4-job.json + output: + count_output: 16 + tool: tests/count-lines19-wf.cwl + id: wf_wc_nomultiple_merge_nested + doc: >- + Test when step source is a single-item list and linkMerge is + listed, then it is wrapped in a list. + tags: [ inline_javascript, workflow ] + +- job: tests/empty.json + output: {count_output: 1} + tool: tests/count-lines5-wf.cwl + id: wf_input_default_missing + doc: Test workflow with default value for input parameter (missing) + tags: [ inline_javascript, workflow ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines5-wf.cwl + id: wf_input_default_provided + doc: Test workflow with default value for input parameter (provided) + tags: [ inline_javascript, workflow ] + +- job: tests/empty.json + output: {default_output: workflow_default} + tool: tests/echo-wf-default.cwl + id: wf_default_tool_default + doc: Test that workflow defaults override tool defaults + tags: [ required, workflow ] + +- job: tests/env-job.json + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-tool1.cwl + id: envvar_req + doc: Test EnvVarRequirement + tags: [ env_var, command_line_tool ] + +- job: tests/scatter-job1.json + output: + out: ["foo one", "foo two", "foo three", "foo four"] + tool: tests/scatter-wf1.cwl + id: wf_scatter_single_param + doc: Test workflow scatter with single scatter parameter + tags: [ scatter, workflow ] + +- job: tests/scatter-job2.json + output: + out: [["foo one three", "foo one four"], ["foo two three", "foo two four"]] + tool: tests/scatter-wf2.cwl + id: wf_scatter_two_nested_crossproduct + doc: Test workflow scatter with two scatter parameters and nested_crossproduct join method + tags: [ scatter, workflow ] + +- job: tests/scatter-job2.json + output: + out: ["foo one three", "foo one four", "foo two three", "foo two four"] + tool: "tests/scatter-wf3.cwl#main" + id: wf_scatter_two_flat_crossproduct + doc: Test workflow scatter with two scatter parameters and flat_crossproduct join method + tags: [ scatter, workflow ] + +- job: tests/scatter-job2.json + output: + out: ["foo one three", "foo two four"] + tool: "tests/scatter-wf4.cwl#main" + id: wf_scatter_two_dotproduct + doc: Test workflow scatter with two scatter parameters and dotproduct join method + tags: [ scatter, workflow ] + +- job: tests/scatter-empty-job1.json + output: + out: [] + tool: tests/scatter-wf1.cwl + id: wf_scatter_emptylist + doc: Test workflow scatter with single empty list parameter + tags: [ scatter, workflow ] + +- job: tests/scatter-empty-job2.json + output: + out: [[], []] + tool: tests/scatter-wf2.cwl + id: wf_scatter_nested_crossproduct_secondempty + doc: Test workflow scatter with two scatter parameters and nested_crossproduct join method with second list empty + tags: [ scatter, workflow ] + +- job: tests/scatter-empty-job3.json + output: + out: [] + tool: "tests/scatter-wf3.cwl#main" + id: wf_scatter_nested_crossproduct_firstempty + doc: Test workflow scatter with two scatter parameters and nested_crossproduct join method with first list empty + tags: [ scatter, workflow ] + +- job: tests/scatter-empty-job2.json + output: + out: [] + tool: "tests/scatter-wf3.cwl#main" + id: wf_scatter_flat_crossproduct_oneempty + doc: Test workflow scatter with two scatter parameters, one of which is empty and flat_crossproduct join method + tags: [ scatter, workflow ] + +- job: tests/scatter-empty-job4.json + output: + out: [] + tool: "tests/scatter-wf4.cwl#main" + id: wf_scatter_dotproduct_twoempty + doc: Test workflow scatter with two empty scatter parameters and dotproduct join method + tags: [ scatter, workflow ] + +- tool: tests/echo-tool.cwl + job: tests/env-job.json + output: + {"out": "hello test env\n"} + id: any_input_param + doc: Test Any type input parameter + tags: [ required, command_line_tool ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines8-wf.cwl + id: nested_workflow + doc: Test nested workflow + tags: [ subworkflow, workflow, inline_javascript ] + +- job: tests/env-job.json + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-wf1.cwl + id: requirement_priority + doc: Test requirement priority + tags: [ env_var, workflow ] + +- job: tests/env-job.json + output: + out: + class: File + checksum: sha1$cdc1e84968261d6a7575b5305945471f8be199b6 + location: out + size: 9 + tool: tests/env-wf2.cwl + id: requirement_override_hints + doc: Test requirements override hints + tags: [ env_var, workflow ] + +- job: tests/env-job.json + output: + out: + class: File + checksum: sha1$cdc1e84968261d6a7575b5305945471f8be199b6 + location: out + size: 9 + tool: tests/env-wf3.cwl + id: requirement_workflow_steps + doc: Test requirements on workflow steps + tags: [ env_var, workflow ] + +- job: tests/empty.json + output: {count_output: 16} + tool: tests/count-lines9-wf.cwl + id: step_input_default_value + doc: Test default value on step input parameter + tags: [ inline_javascript, workflow ] + +- job: tests/empty.json + output: {count_output: 16} + tool: tests/count-lines11-wf.cwl + id: step_input_default_value_nosource + doc: Test use default value on step input parameter with empty source + tags: [ inline_javascript, workflow ] + +- job: tests/file1-null.json + output: {count_output: 16} + tool: tests/count-lines11-wf.cwl + id: step_input_default_value_nullsource + doc: Test use default value on step input parameter with null source + tags: [ inline_javascript, workflow ] + +- job: tests/cat-job.json + output: {count_output: 1} + tool: tests/count-lines11-wf.cwl + id: step_input_default_value_overriden + doc: Test default value on step input parameter overridden by provided source + tags: [ inline_javascript, workflow ] + +- job: tests/revsort-job.json + output: + output: + class: File + checksum: sha1$b9214658cc453331b62c2282b772a5c063dbd284 + location: output.txt + size: 1111 + tool: tests/revsort.cwl + id: wf_simple + doc: Test simple workflow + tags: [ required, workflow ] + +- job: tests/cat-job.json + output: + output_file: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + location: output.txt + size: 13 + tool: tests/cat5-tool.cwl + id: hints_unknown_ignored + doc: Test unknown hints are ignored. + tags: [ required, command_line_tool ] + +- job: tests/search-job.json + output: + outfile: + class: File + checksum: sha1$e2dc9daaef945ac15f01c238ed2f1660f60909a0 + location: result.txt + size: 142 + indexedfile: { + "location": "input.txt", + "class": "File", + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376", + "secondaryFiles": [ + { + "location": "input.txt.idx1", + "class": "File", + "checksum": "sha1$1f6fe811644355974cdd06d9eb695d6e859f3b44", + "size": 1500 + }, + { + "location": "input.idx2", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.txt.idx3", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.txt.idx4", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.txt.idx5", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "location": "input.idx6.txt", + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "size": 0 + }, + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "input.txt.idx7", + "size": 0 + }, + { + "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b", + "class": "File", + "location": "hello.txt", + "size": 13 + }, + { + "class": "Directory", + "listing": [{ + "basename": "index", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "index", + "size": 0 + }], + "location": "input.txt_idx8", + } + ], + "size": 1111 + } + tool: "tests/search.cwl#main" + id: initial_workdir_secondary_files_expr + doc: >- + Test InitialWorkDirRequirement linking input files and capturing secondaryFiles + on input and output. Also tests the use of a variety of parameter references + and expressions in the secondaryFiles field. + tags: [ initial_work_dir, inline_javascript, workflow ] + +- job: tests/rename-job.json + output: + outfile: + class: File + checksum: sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376 + location: fish.txt + size: 1111 + tool: tests/rename.cwl + id: rename + doc: >- + Test InitialWorkDirRequirement with expression in filename. + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/string-job.json + output: + out: + class: File + checksum: sha1$6a47aa22b2a9d13a66a24b3ee5eaed95ce4753cf + location: example.conf + size: 16 + tool: tests/iwdr-entry.cwl + id: initial_workdir_trailingnl + doc: Test if trailing newline is present in file entry in InitialWorkDir + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/wc-job.json + output: + output: 16 + tool: tests/wc4-tool.cwl + id: inline_expressions + doc: >- + Test inline expressions + tags: [ inline_javascript, command_line_tool ] + +- job: tests/schemadef-job.json + output: + output: + location: output.txt + size: 12 + class: File + checksum: "sha1$f12e6cfe70f3253f70b0dbde17c692e7fb0f1e5e" + tool: tests/schemadef-tool.cwl + id: schemadef_req_tool_param + doc: >- + Test SchemaDefRequirement definition used in tool parameter + tags: [ schema_def, command_line_tool ] + +- job: tests/schemadef-job.json + output: + output: + location: output.txt + size: 12 + class: File + checksum: "sha1$f12e6cfe70f3253f70b0dbde17c692e7fb0f1e5e" + tool: tests/schemadef-wf.cwl + id: schemadef_req_wf_param + doc: >- + Test SchemaDefRequirement definition used in workflow parameter + tags: [ schema_def, workflow ] + +- job: tests/empty.json + output: { + "t1": { + "bar": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + } + }, + "t10": true, + "t11": true, + "t12": null, + "t13": "-zab1", + "t14": "-zab1", + "t15": "-zab1", + "t16": "-zab1", + "t17": "zab1 zab1", + "t18": "zab1 zab1", + "t19": "zab1 zab1", + "t2": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + }, + "t20": "zab1 zab1", + "t21": "2 2", + "t22": "true true", + "t23": "true true", + "t24": "null null", + "t25": "b", + "t26": "b b", + "t3": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + }, + "t4": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + }, + "t5": "zab1", + "t6": "zab1", + "t7": "zab1", + "t8": "zab1", + "t9": 2, + "t27": null, + "t28": 3 + } + tool: tests/params.cwl + id: param_evaluation_noexpr + doc: Test parameter evaluation, no support for JS expressions + tags: [ required, command_line_tool ] + +- job: tests/empty.json + output: { + "t1": { + "bar": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + } + }, + "t10": true, + "t11": true, + "t12": null, + "t13": "-zab1", + "t14": "-zab1", + "t15": "-zab1", + "t16": "-zab1", + "t17": "zab1 zab1", + "t18": "zab1 zab1", + "t19": "zab1 zab1", + "t2": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + }, + "t20": "zab1 zab1", + "t21": "2 2", + "t22": "true true", + "t23": "true true", + "t24": "null null", + "t25": "b", + "t26": "b b", + "t3": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + }, + "t4": { + "b az": 2, + "b\"az": null, + "b'az": true, + "baz": "zab1", + "buz": [ + "a", + "b", + "c" + ] + }, + "t5": "zab1", + "t6": "zab1", + "t7": "zab1", + "t8": "zab1", + "t9": 2, + "t27": null, + "t28": 3 + } + tool: tests/params2.cwl + id: param_evaluation_expr + doc: >- + Test parameter evaluation, with support for JS expressions + tags: [ inline_javascript, command_line_tool ] + +- output: {} + job: tests/cat-job.json + tool: tests/metadata.cwl + id: metadata + doc: Test metadata + tags: [ required, command_line_tool ] + +- job: tests/formattest-job.json + output: + output: + "location": "output.txt" + "format": "http://edamontology.org/format_2330" + "size": 1111 + "class": "File" + "checksum": "sha1$97fe1b50b4582cebc7d853796ebd62e3e163aa3f" + tool: tests/formattest.cwl + id: format_checking + doc: >- + Test simple format checking. + tags: [ required, command_line_tool ] + +- job: tests/formattest2-job.json + output: + output: + "location": "output.txt" + "format": "http://edamontology.org/format_1929" + "size": 12010 + "class": "File" + "checksum": "sha1$971d88faeda85a796752ecf752b7e2e34f1337ce" + tool: tests/formattest2.cwl + id: format_checking_subclass + doc: >- + Test format checking against ontology using subclassOf. + tags: [ required, command_line_tool ] + +- job: tests/formattest2-job.json + output: + output: + "location": "output.txt" + "format": "http://edamontology.org/format_1929" + "size": 12010 + "class": "File" + "checksum": "sha1$971d88faeda85a796752ecf752b7e2e34f1337ce" + tool: tests/formattest3.cwl + id: format_checking_equivalentclass + doc: >- + Test format checking against ontology using equivalentClass. + tags: [ required, command_line_tool ] + +- tool: tests/optional-output.cwl + job: tests/cat-job.json + output: + optional_file: null + output_file: + location: output.txt + size: 13 + class: "File" + checksum: "sha1$47a013e660d408619d894b20806b1d5086aab03b" + id: output_secondaryfile_optional + doc: >- + Test optional output file and optional secondaryFile on output. + tags: [ docker, command_line_tool ] + +- job: tests/empty.json + output: + out: "\n" + tool: tests/vf-concat.cwl + id: valuefrom_ignored_null + doc: Test that valueFrom is ignored when the parameter is null + tags: [ inline_javascript, command_line_tool ] + +- job: tests/cat-job.json + output: + out: "a string\n" + tool: tests/vf-concat.cwl + id: valuefrom_secondexpr_ignored + doc: Test that second expression in concatenated valueFrom is not ignored + tags: [ inline_javascript, command_line_tool ] + +- job: tests/step-valuefrom-wf.json + output: {count_output: 16} + tool: tests/step-valuefrom-wf.cwl + id: valuefrom_wf_step + doc: Test valueFrom on workflow step. + tags: [ step_input, inline_javascript, workflow ] + +- job: tests/step-valuefrom-job.json + output: {val: "3\n"} + tool: tests/step-valuefrom2-wf.cwl + id: valuefrom_wf_step_multiple + doc: Test valueFrom on workflow step with multiple sources + tags: [ step_input, inline_javascript, multiple_input, workflow ] + +- job: tests/step-valuefrom-job.json + output: {val: "3\n"} + tool: tests/step-valuefrom3-wf.cwl + id: valuefrom_wf_step_other + doc: Test valueFrom on workflow step referencing other inputs + tags: [ step_input, inline_javascript, workflow ] + +- job: tests/record-output-job.json + output: + "orec": { + "ofoo": { + "location": "foo", + "size": 1111, + "class": "File", + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376" + }, + "obar": { + "location": "bar", + "size": 12010, + "class": "File", + "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118" + } + } + tool: tests/record-output.cwl + id: record_output_binding + doc: Test record type output binding. + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + output: { + "foo": { + "location": "foo", + "class": "File", + "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15", + "size": 4 + } + } + tool: tests/test-cwl-out.cwl + id: docker_json_output_path + doc: >- + Test support for reading cwl.output.json when running in a Docker container + and just 'path' is provided. + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + output: { + "foo": { + "location": "foo", + "class": "File", + "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15", + "size": 4 + } + } + tool: tests/test-cwl-out2.cwl + id: docker_json_output_location + doc: >- + Test support for reading cwl.output.json when running in a Docker container + and just 'location' is provided. + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + output: { + "foo": { + "location": "foo", + "class": "File", + "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15", + "size": 4 + } + } + tool: tests/test-cwl-out3.cwl + id: json_output_path_relative + doc: >- + Test support for reading cwl.output.json where 'path' is relative path in output dir. + tags: [ required, command_line_tool ] + +- job: tests/empty.json + output: { + "foo": { + "location": "foo", + "class": "File", + "checksum": "sha1$f1d2d2f924e986ac86fdf7b36c94bcdf32beec15", + "size": 4 + } + } + tool: tests/test-cwl-out4.cwl + id: json_output_location_relative + doc: >- + Test support for reading cwl.output.json where 'location' is relative reference to output dir. + tags: [ required, command_line_tool ] + +- job: tests/abc.json + output: + files: [{ + "location": "a", + "size": 0, + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" + }, + { + "location": "b", + "size": 0, + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" + }, + { + "location": "c", + "size": 0, + "class": "File", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" + }] + tool: tests/glob-expr-list.cwl + id: multiple_glob_expr_list + doc: Test support for returning multiple glob patterns from expression + tags: [ required, command_line_tool ] + +- job: tests/scatter-valuefrom-job1.json + output: + out: ["foo one one", "foo one two", "foo one three", "foo one four"] + tool: tests/scatter-valuefrom-wf1.cwl + id: wf_scatter_oneparam_valuefrom + doc: Test workflow scatter with single scatter parameter and two valueFrom on step input (first and current el) + tags: [ scatter, step_input, workflow ] + +- job: tests/scatter-valuefrom-job2.json + output: + out: [["foo one one three", "foo one one four"], ["foo one two three", "foo one two four"]] + tool: tests/scatter-valuefrom-wf2.cwl + id: wf_scatter_twoparam_nested_crossproduct_valuefrom + doc: Test workflow scatter with two scatter parameters and nested_crossproduct join method and valueFrom on step input + tags: [ scatter, step_input, workflow ] + +- job: tests/scatter-valuefrom-job2.json + output: + out: ["foo one one three", "foo one one four", "foo one two three", "foo one two four"] + tool: "tests/scatter-valuefrom-wf3.cwl#main" + id: wf_scatter_twoparam_flat_crossproduct_valuefrom + doc: Test workflow scatter with two scatter parameters and flat_crossproduct join method and valueFrom on step input + tags: [ scatter, step_input, workflow ] + +- job: tests/scatter-valuefrom-job2.json + output: + out: ["foo one one three", "foo one two four"] + tool: "tests/scatter-valuefrom-wf4.cwl#main" + id: wf_scatter_twoparam_dotproduct_valuefrom + doc: Test workflow scatter with two scatter parameters and dotproduct join method and valueFrom on step input + tags: [ scatter, step_input, workflow ] + +- job: tests/scatter-valuefrom-job1.json + output: + out: ["foo one one", "foo two two", "foo three three", "foo four four"] + tool: tests/scatter-valuefrom-wf5.cwl + id: wf_scatter_oneparam_valuefrom_twice_current_el + doc: Test workflow scatter with single scatter parameter and two valueFrom on step input (current el twice) + tags: [ scatter, step_input, workflow ] + +- job: tests/scatter-valuefrom-job3.json + tool: tests/scatter-valuefrom-wf6.cwl + id: wf_scatter_oneparam_valueFrom + doc: Test valueFrom eval on scattered input parameter + output: + out_message: [ + { + "checksum": "sha1$98030575f6fc40e5021be5a8803a6bef94aee11f", + "location": Any, + "class": "File", + "size": 16 + }, + { + "checksum": "sha1$edcacd50778d98ae113015406b3195c165059dd8", + "location": Any, + "class": "File", + "size": 16 + } + ] + tags: [ scatter, step_input, workflow ] + +- job: tests/conflict-job.json + output: { + "fileout": { + "location": "out.txt", + "checksum": "sha1$a2d8d6e7b28295dc9977dc3bdb652ddd480995f0", + "class": "File", + "size": 25 + } + } + tool: "tests/conflict-wf.cwl#collision" + id: wf_two_inputfiles_namecollision + doc: Test workflow two input files with same name. + tags: [ required, workflow ] + +- job: tests/dir-job.yml + output: + "outlist": { + "size": 20, + "location": "output.txt", + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "class": "File" + } + tool: tests/dir.cwl + id: directory_input_param_ref + doc: Test directory input with parameter reference + tags: [ shell_command, command_line_tool ] + +- job: tests/dir-job.yml + output: + "outlist": { + "size": 20, + "location": "output.txt", + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "class": "File" + } + tool: tests/dir2.cwl + id: directory_input_docker + doc: Test directory input in Docker + tags: [ command_line_tool, shell_command ] + +- job: tests/dir3-job.yml + output: + "outdir": { + "class": "Directory", + "listing": [ + { + "class": "File", + "location": "goodbye.txt", + "checksum": "sha1$dd0a4c4c49ba43004d6611771972b6cf969c1c01", + "size": 24 + }, + { + "class": "File", + "location": "hello.txt", + "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b", + "size": 13 + } + ], + } + tool: tests/dir3.cwl + id: directory_output + doc: Test directory output + tags: [ required, command_line_tool ] + +- job: tests/dir4-job.yml + output: { + "outlist": { + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "size": 20, + "location": "output.txt", + "class": "File" + } + } + tool: tests/dir4.cwl + id: directory_secondaryfiles + doc: Test directories in secondaryFiles + tags: [ shell_command, command_line_tool ] + +- job: tests/dir-job.yml + output: { + "outlist": { + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "size": 20, + "location": "output.txt", + "class": "File" + } + } + tool: tests/dir5.cwl + id: dynamic_initial_workdir + doc: Test dynamic initial work dir + tags: [ shell_command, initial_work_dir, command_line_tool ] + +- job: tests/stagefile-job.yml + output: { + "outfile": { + "checksum": "sha1$b769c7b2e316edd4b5eb2d24799b2c1f9d8c86e6", + "size": 1111, + "location": "bob.txt", + "class": "File" + } + } + tool: tests/stagefile.cwl + id: writable_stagedfiles + doc: Test writable staged files. + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/file-literal.yml + output: + output_file: + class: File + checksum: sha1$d0e04ff6c413c7d57f9a0ca0a33cd3ab52e2dd9c + location: output.txt + size: 18 + tool: tests/cat3-tool.cwl + id: input_file_literal + doc: Test file literal as input + tags: [ required, command_line_tool ] + +- job: tests/arguments-job.yml + output: + classfile: + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: Hello.class + class: File + size: 0 + tool: tests/linkfile.cwl + id: initial_workdir_expr + doc: Test expression in InitialWorkDir listing + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/wc-job.json + output: + b: + checksum: sha1$c4cfd130e7578714e3eef91c1d6d90e0e0b9db3e + location: whale.xtx + class: File + size: 21 + tool: tests/nameroot.cwl + id: nameroot_nameext_stdout_expr + doc: Test nameroot/nameext expression in arguments, stdout + tags: [ required, command_line_tool ] + +- job: tests/dir-job.yml + output: + "outlist": { + "size": 20, + "location": "output.txt", + "checksum": "sha1$13cda8661796ae241da3a18668fb552161a72592", + "class": "File" + } + tool: tests/dir6.cwl + id: input_dir_inputbinding + doc: Test directory input with inputBinding + tags: [ shell_command, command_line_tool ] + +- job: tests/nested-array-job.yml + output: + echo: + checksum: sha1$3f786850e387550fdab836ed7e6dc881de23001b + location: echo.txt + class: File + size: 2 + tool: tests/nested-array.cwl + id: cl_gen_arrayofarrays + doc: Test command line generation of array-of-arrays + tags: [ required, command_line_tool ] + +- job: tests/empty.json + output: {} + tool: tests/envvar.cwl + id: env_home_tmpdir + doc: Test $HOME and $TMPDIR are set correctly + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + output: {} + tool: tests/envvar2.cwl + id: env_home_tmpdir_docker + doc: Test $HOME and $TMPDIR are set correctly in Docker + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + output: + "out": { + "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a", + "location": "whatever.txt", + "class": "File", + "size": 2 + } + tool: "tests/js-expr-req-wf.cwl#wf" + id: expressionlib_tool_wf_override + doc: Test that expressionLib requirement of individual tool step overrides expressionLib of workflow. + tags: [ inline_javascript, workflow ] + +- job: tests/initialworkdirrequirement-docker-out-job.json + output: + OUTPUT: + "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118" + "location": "ref.fasta" + "secondaryFiles": [{ + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "ref.fasta.fai", + "class": "File", + "size": 0 + }] + "class": "File" + "size": 12010 + tool: tests/initialworkdirrequirement-docker-out.cwl + id: initial_workdir_output + doc: Test output of InitialWorkDir + tags: [ docker, initial_work_dir, command_line_tool ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines10-wf.cwl + id: embedded_subworkflow + doc: Test embedded subworkflow + tags: [ subworkflow, workflow ] + +- job: tests/docker-array-secondaryfiles-job.json + output: { + "bai_list": { + "checksum": "sha1$081fc0e57d6efa5f75eeb237aab1d04031132be6", + "location": "fai.list", + "class": "File", + "size": 386 + } + } + tool: tests/docker-array-secondaryfiles.cwl + id: filesarray_secondaryfiles + doc: Test required, optional and null secondaryFiles on array of files. + tags: [ docker, inline_javascript, shell_command, command_line_tool ] + +- job: tests/docker-array-secondaryfiles-job2.json + should_fail: true + tool: tests/docker-array-secondaryfiles.cwl + id: filesarray_secondaryfiles2 + doc: Test required, optional and null secondaryFiles on array of files. + tags: [ docker, inline_javascript, shell_command, command_line_tool ] + +- job: tests/dir7.yml + output: { + "dir": { + "location": "a_directory", + "class": "Directory", + "listing": [ + { + "class": "File", + "location": "whale.txt", + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376", + "size": 1111 + }, + { + "class": "File", + "location": "hello.txt", + "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b", + "size": 13 + } + ] + } + } + tool: tests/dir7.cwl + id: exprtool_directory_literal + doc: Test directory literal output created by ExpressionTool + tags: [ inline_javascript, expression_tool ] + +- job: tests/empty.json + output: + lit: + location: "a_file" + class: "File" + checksum: "sha1$fea23663b9c8ed71968f86415b5ec091bb111448" + size: 19 + tool: tests/file-literal-ex.cwl + id: exprtool_file_literal + doc: Test file literal output created by ExpressionTool + tags: [ inline_javascript, expression_tool ] + +- job: tests/empty.json + output: + "thing": { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "thing", + "class": "File", + "size": 0 + } + tool: tests/docker-output-dir.cwl + id: dockeroutputdir + doc: Test dockerOutputDirectory + tags: [ docker, command_line_tool ] + +- job: tests/empty.json + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/imported-hint.cwl + id: hints_import + doc: Test hints with $import + tags: [ required, command_line_tool ] + +- output: {} + job: tests/default_path_job.yml + tool: tests/default_path.cwl + id: default_path_notfound_warning + doc: Test warning instead of error when default path is not found + tags: [ required, command_line_tool ] + +- output: + args: [-A,'2',-B,baz,-C,'10','9','8','7','6','5','4','3','2','1',-D] + job: tests/empty.json + tool: tests/inline-js.cwl + id: inlinejs_req_expressions + doc: Test InlineJavascriptRequirement with multiple expressions in the same tool + tags: [ inline_javascript, command_line_tool ] + +- job: tests/recursive-input-directory.yml + output: + output_dir: { + "basename": "work_dir", + "class": "Directory", + "listing": [ + { + "basename": "a", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/a", + "size": 0 + }, + { + "basename": "b", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/b", + "size": 0 + }, + { + "basename": "c", + "class": "Directory", + "listing": [ + { + "basename": "d", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/c/d", + "size": 0 + } + ], + "location": "work_dir/c", + }, + { + "basename": "e", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "work_dir/e", + "size": 0 + }, + ], + "location": "work_dir", + } + test_result: { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "output.txt", + "size": 0 + } + tool: tests/recursive-input-directory.cwl + id: input_dir_recurs_copy_writable + doc: Test if a writable input directory is recursively copied and writable + tags: [ initial_work_dir, shell_command, command_line_tool ] + +- output: + out: "t\n" + job: tests/empty.json + tool: tests/null-defined.cwl + id: null_missing_params + doc: Test that missing parameters are null (not undefined) in expression + tags: [ inline_javascript, command_line_tool ] + +- output: + out: "f\n" + job: tests/cat-job.json + tool: tests/null-defined.cwl + id: param_notnull_expr + doc: Test that provided parameter is not null in expression + tags: [ inline_javascript, command_line_tool ] + +- job: tests/revsort-job.json + output: + output: + class: File + checksum: sha1$b9214658cc453331b62c2282b772a5c063dbd284 + location: output.txt + size: 1111 + tool: tests/revsort-packed.cwl#main + id: wf_compound_doc + doc: Test compound workflow document + tags: [ required, workflow ] + +- job: tests/basename-fields-job.yml + output: + extFile: + checksum: sha1$301a72c82a835e1737caf30f94d0eec210c4d9f1 + class: File + size: 5 + location: Any + path: Any + rootFile: + checksum: sha1$b4a583c391e234cf210e1d576f68f674c8ad7ecd + class: File + size: 10 + location: Any + path: Any + tool: tests/basename-fields-test.cwl + id: nameroot_nameext_generated + doc: Test that nameroot and nameext are generated from basename at execution time by the runner + tags: [ step_input, workflow ] + +- job: tests/wc-job.json + output: {} + tool: tests/initialwork-path.cwl + id: initialworkpath_output + doc: Test that file path in $(inputs) for initialworkdir is in $(outdir). + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/count-lines6-job.json + output: + count_output: 34 + tool: tests/count-lines12-wf.cwl + id: wf_scatter_twopar_oneinput_flattenedmerge + doc: >- + Test single step workflow with Scatter step and two data links connected to + same input, flattened merge behavior. Workflow inputs are set as list + tags: [ multiple_input, inline_javascript, workflow ] + +- job: tests/sum-job.json + output: + result: 12 + tool: tests/sum-wf.cwl + id: wf_multiplesources_multipletypes + doc: Test step input with multiple sources with multiple types + tags: [ step_input, inline_javascript, multiple_input, workflow ] + +- job: tests/empty.json + output: { + "stderr_file": { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": Any, + "class": "File", + "size": 0 + }, + "stdout_file": { + "checksum": "sha1$1555252d52d4ec3262538a4426a83a99cfff4402", + "location": Any, + "class": "File", + "size": 9 + } + } + tool: tests/shellchar.cwl + id: shelldir_notinterpreted + doc: "Test that shell directives are not interpreted." + tags: [ required, command_line_tool ] + +- job: tests/empty.json + output: { + "stderr_file": { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": Any, + "class": "File", + "size": 0 + }, + "stdout_file": { + "checksum": "sha1$1555252d52d4ec3262538a4426a83a99cfff4402", + "location": Any, + "class": "File", + "size": 9 + } + } + tool: tests/shellchar2.cwl + id: shelldir_quoted + doc: "Test that shell directives are quoted." + tags: [ shell_command, command_line_tool ] + +- job: tests/empty.json + output: + out: { + "basename": "emptyWritableDir", + "listing": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "basename": "blurg", + "location": "blurg", + "class": "File", + "size": 0 + } + ], + "location": "emptyWritableDir", + "class": "Directory" + } + tool: tests/writable-dir.cwl + id: initial_workdir_empty_writable + doc: Test empty writable dir with InitialWorkDirRequirement + tags: [ inline_javascript, initial_work_dir, command_line_tool ] + +- job: tests/empty.json + output: + out: { + "basename": "emptyWritableDir", + "listing": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "basename": "blurg", + "location": "blurg", + "class": "File", + "size": 0 + } + ], + "location": "emptyWritableDir", + "class": "Directory" + } + tool: tests/writable-dir-docker.cwl + id: initial_workdir_empty_writable_docker + doc: Test empty writable dir with InitialWorkDirRequirement inside Docker + tags: [ inline_javascript, initial_work_dir, command_line_tool ] + +- job: tests/dynresreq-job.yaml + tool: tests/dynresreq.cwl + id: dynamic_resreq_inputs + doc: Test dynamic resource reqs referencing inputs + output: + output: { + "location": "cores.txt", + "size": 2, + "class": "File", + "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a" + } + tags: [ resource, command_line_tool ] + +- job: tests/file-literal.yml + output: + output_file: + class: File + checksum: sha1$d0e04ff6c413c7d57f9a0ca0a33cd3ab52e2dd9c + location: output.txt + size: 18 + tool: tests/cat3-nodocker.cwl + id: fileliteral_input_docker + doc: Test file literal as input without Docker + tags: [ required, command_line_tool ] + +- doc: Test that OutputBinding.glob is sorted as specified by POSIX + job: tests/empty.json + id: outputbinding_glob_sorted + tool: tests/glob_test.cwl + output: + letters: + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: a + class: File + size: 0 + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: b + class: File + size: 0 + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: c + class: File + size: 0 + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: w + class: File + size: 0 + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: x + class: File + size: 0 + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: y + class: File + size: 0 + - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: z + class: File + size: 0 + tags: [ required, command_line_tool ] + +- doc: Test InitialWorkDirRequirement with a nested directory structure from another step + job: tests/empty.json + output: + ya_empty: + class: File + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + location: ya + size: 0 + tool: tests/iwdr_with_nested_dirs.cwl + id: initialworkdir_nesteddir + tags: [ initial_work_dir, workflow ] + +- job: tests/bool-empty-inputbinding-job.json + output: { + "args": [ + ] + } + tool: tests/bool-empty-inputbinding.cwl + id: booleanflags_cl_noinputbinding + doc: "Test that boolean flags do not appear on command line if inputBinding is empty and not null" + tags: [ required, command_line_tool ] + +- job: tests/empty.json + output: { + "args": [] + } + tool: tests/stage-unprovided-file.cwl + id: expr_reference_self_noinput + doc: Test that expression engine does not fail to evaluate reference to self + with unprovided input + tags: [ required, command_line_tool ] + +- tool: tests/exit-success.cwl + id: success_codes + doc: Test successCodes + job: tests/empty.json + output: {} + tags: [ required, command_line_tool ] + +- job: tests/dynresreq-job.yaml + doc: Test simple workflow with a dynamic resource requirement + tool: tests/dynresreq-workflow.cwl + id: dynamic_resreq_wf + output: + cores: { + "location": "output", + "size": 2, + "class": "File", + "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a" + } + tags: [ resource, workflow ] + +- job: tests/empty-array-job.json + output: { + "args": [] + } + tool: tests/empty-array-input.cwl + id: cl_empty_array_input + doc: "Test that empty array input does not add anything to command line" + tags: [ required, command_line_tool ] + +- job: tests/empty.json + tool: tests/steplevel-resreq.cwl + id: resreq_step_overrides_wf + doc: Test that ResourceRequirement on a step level redefines requirement on the workflow level + output: + out: { + "location": "cores.txt", + "size": 2, + "class": "File", + "checksum": "sha1$e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e" + } + tags: [ resource, workflow ] + +- job: tests/array-of-strings-job.yml + output: { + "args": ["replacementValue"] + } + tool: tests/valueFrom-constant.cwl + id: valuefrom_constant_overrides_inputs + doc: Test valueFrom with constant value overriding provided array inputs + tags: [ required, command_line_tool ] + +- job: tests/dynresreq-dir-job.yaml + tool: tests/dynresreq-dir.cwl + id: dynamic_resreq_filesizes + doc: Test dynamic resource reqs referencing the size of Files inside a Directory + output: + output: { + "location": "cores.txt", + "size": 2, + "class": "File", + "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a" + } + tags: [ resource, command_line_tool ] + +- job: tests/empty.json + tool: tests/pass-unconnected.cwl + id: wf_step_connect_undeclared_param + doc: >- + Test that it is not an error to connect a parameter to a workflow + step, even if the parameter doesn't appear in the `run` process + inputs. + output: { "out": "hello inp1\n" } + tags: [ required, workflow ] + +- job: tests/empty.json + tool: tests/fail-unconnected.cwl + id: wf_step_access_undeclared_param + doc: >- + Test that parameters that don't appear in the `run` process + inputs are not present in the input object used to run the tool. + should_fail: true + tags: [ required, workflow ] + +- job: tests/empty.json + output: + results: + basename: results + class: File + checksum: "sha1$7d5ca8c0c03e883c56c4eb1ef6f6bb9bccad4d86" + size: 8 + tool: tests/envvar3.cwl + id: env_home_tmpdir_docker_no_return_code + doc: Test $HOME and $TMPDIR are set correctly in Docker without using return code + tags: [ shell_command, command_line_tool ] + +- job: tests/scatter-valuefrom-job1.json + output: + out: ["foo one one", "foo one two", "foo one three", "foo one four"] + tool: tests/scatter-valuefrom-inputs-wf1.cwl + id: wf_scatter_oneparam_valuefrom_inputs + doc: Test workflow scatter with single scatter parameter and two valueFrom using $inputs (first and current el) + tags: [ scatter, step_input, workflow ] + +- job: tests/import_schema-def_job.yml + output: + output_bam: + checksum: "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709" + basename: a.bam + class: File + size: 0 + tool: tests/import_schema-def_packed.cwl + id: packed_import_schema + doc: SchemaDefRequirement with $import, and packed + tags: [ schema_def, workflow ] + +- job: tests/dir4-subdir-1-job.yml + output: { + "outlist": { + "checksum": "sha1$9d9bc8f5252d39274b5dfbac64216c6e888f5dfc", + "size": 14, + "location": "output.txt", + "class": "File" + } + } + tool: tests/dir4.cwl + id: job_input_secondary_subdirs + doc: Test specifying secondaryFiles in subdirectories of the job input document. + tags: [ shell_command, command_line_tool ] + +- job: tests/dir4-subdir-2-job.yml + output: { + "outlist": { + "checksum": "sha1$9d9bc8f5252d39274b5dfbac64216c6e888f5dfc", + "size": 14, + "location": "output.txt", + "class": "File" + } + } + tool: tests/dir4.cwl + id: job_input_subdir_primary_and_secondary_subdirs + doc: Test specifying secondaryFiles in same subdirectory of the job input as the primary input file. + tags: [ shell_command, command_line_tool ] + +- job: tests/count-lines3-job.json + output: + count_output: [16, 1] + tool: tests/count-lines18-wf.cwl + id: scatter_embedded_subworkflow + doc: Test simple scatter over an embedded subworkflow + tags: [ workflow, inline_javascript ] + +- job: tests/count-lines4-job.json + output: + count_output: [16, 1] + tool: tests/count-lines14-wf.cwl + id: scatter_multi_input_embedded_subworkflow + doc: Test simple multiple input scatter over an embedded subworkflow + tags: [ workflow, scatter, subworkflow, multiple_input, inline_javascript ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines15-wf.cwl + doc: Test twice nested subworkflow + id: workflow_embedded_subworkflow_embedded_subsubworkflow + tags: [ workflow, subworkflow, inline_javascript ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines16-wf.cwl + id: workflow_embedded_subworkflow_with_tool_and_subsubworkflow + doc: Test subworkflow of mixed depth with tool first + tags: [ workflow, subworkflow, inline_javascript ] + +- job: tests/wc-job.json + output: {count_output: 16} + tool: tests/count-lines17-wf.cwl + id: workflow_embedded_subworkflow_with_subsubworkflow_and_tool + doc: Test subworkflow of mixed depth with tool after + tags: [ workflow, subworkflow, inline_javascript ] + +- job: tests/record-output-job.json + output: + "orec": { + "ofoo": { + "location": "foo", + "size": 1111, + "class": "File", + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376" + }, + "obar": { + "location": "bar", + "size": 12010, + "class": "File", + "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118" + } + } + tool: tests/record-output-wf.cwl + id: workflow_records_inputs_and_outputs + doc: Test record type inputs to and outputs from workflows. + tags: [ workflow, shell_command ] + +- job: tests/io-int.json + output: {"o": 10} + tool: tests/io-int-wf.cwl + id: workflow_integer_input + doc: Test integer workflow input and outputs + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/io-int.json + output: {"o": 10} + tool: tests/io-int-optional-wf.cwl + id: workflow_integer_input_optional_specified + doc: Test optional integer workflow inputs (specified) + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/empty.json + output: {"o": 4} + tool: tests/io-int-optional-wf.cwl + id: workflow_integer_input_optional_unspecified + doc: Test optional integer workflow inputs (unspecified) + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/io-int.json + output: {"o": 10} + tool: tests/io-int-default-wf.cwl + id: workflow_integer_input_default_specified + doc: Test default integer workflow inputs (specified) + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/empty.json + output: {"o": 8} + tool: tests/io-int-default-wf.cwl + id: workflow_integer_input_default_unspecified + doc: Test default integer workflow inputs (unspecified) + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/empty.json + output: {"o": 13} + tool: tests/io-int-default-tool-and-wf.cwl + id: workflow_integer_input_default_and_tool_integer_input_default + doc: Test default integer tool and workflow inputs (unspecified) + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/empty.json + output: {"o": { + "class": "File", + "basename": "output", + "size": 1111, + "location": Any, + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376" + }} + tool: tests/io-file-default-wf.cwl + id: workflow_file_input_default_unspecified + doc: Test File input with default unspecified to workflow + tags: [ workflow ] + +- job: tests/default_path_job.yml + output: {"o": { + "class": "File", + "basename": "output", + "location": Any, + "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b", + "size": 13 + }} + tool: tests/io-file-default-wf.cwl + id: workflow_file_input_default_specified + doc: Test File input with default specified to workflow + tags: [ workflow ] + +- job: tests/job-input-array-one-empty-file.json + output: {"output_file": + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "basename": "output.txt", + "location": Any, + "class": "File", + "size": 0 + }} + tool: tests/io-file-or-files.cwl + id: clt_optional_union_input_file_or_files_with_array_of_one_file_provided + doc: Test input union type or File or File array to a tool with one file in array specified. + tags: [ command_line_tool, inline_javascript ] + +- job: tests/job-input-array-few-files.json + output: {"output_file": + { + "checksum": "sha1$6d1723861ad5a1260f1c3c07c93076c5a215f646", + "basename": "output.txt", + "location": Any, + "class": "File", + "size": 1114 + }} + tool: tests/io-file-or-files.cwl + id: clt_optional_union_input_file_or_files_with_many_files_provided + doc: Test input union type or File or File array to a tool with a few files in array specified. + tags: [ command_line_tool, inline_javascript ] + +- job: tests/job-input-one-file.json + output: {"output_file": + { + "checksum": "sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376", + "basename": "output.txt", + "location": Any, + "class": "File", + "size": 1111 + }} + tool: tests/io-file-or-files.cwl + id: clt_optional_union_input_file_or_files_with_single_file_provided + doc: Test input union type or File or File array to a tool with one file specified. + tags: [ command_line_tool, inline_javascript ] + +- job: tests/job-input-null.json + output: {"output_file": + { + "checksum": "sha1$503458abf7614be3fb26d85ff5d8f3e17aa0a552", + "basename": "output.txt", + "location": Any, + "class": "File", + "size": 10 + }} + tool: tests/io-file-or-files.cwl + id: clt_optional_union_input_file_or_files_with_nothing_provided + doc: Test input union type or File or File array to a tool with null specified. + tags: [ command_line_tool, inline_javascript ] + +- job: tests/io-any-int.json + output: {"t1": 7} + tool: tests/io-any-1.cwl + id: clt_any_input_with_integer_provided + doc: Test Any parameter with integer input to a tool + tags: [ command_line_tool, inline_javascript ] + +- job: tests/io-any-string.json + output: {"t1": "7"} + tool: tests/io-any-1.cwl + id: clt_any_input_with_string_provided + doc: Test Any parameter with string input to a tool + tags: [ command_line_tool, inline_javascript ] + +- job: tests/io-any-file.json + output: {"t1": "File"} + tool: tests/io-any-1.cwl + id: clt_any_input_with_file_provided + doc: Test Any parameter with file input to a tool + tags: [ command_line_tool, inline_javascript ] + +- job: tests/io-any-array.json + output: {"t1": [1, "moocow"]} + tool: tests/io-any-1.cwl + id: clt_any_input_with_mixed_array_provided + doc: Test Any parameter with array input to a tool + tags: [ command_line_tool, inline_javascript ] + +- job: tests/io-any-record.json + output: {"t1": {"moo": 1, "cow": 5}} + tool: tests/io-any-1.cwl + id: clt_any_input_with_record_provided + doc: Test Any parameter with record input to a tool + tags: [ command_line_tool, inline_javascript ] + +- job: tests/io-any-int.json + output: {"t1": 7} + tool: tests/io-any-wf-1.cwl + id: workflow_any_input_with_integer_provided + doc: Test Any parameter with integer input to a workflow + tags: [ workflow, inline_javascript ] + +- job: tests/io-any-string.json + output: {"t1": "7"} + tool: tests/io-any-wf-1.cwl + id: workflow_any_input_with_string_provided + doc: Test Any parameter with string input to a workflow + tags: [ workflow, inline_javascript ] + +- job: tests/io-any-file.json + output: {"t1": "File"} + tool: tests/io-any-wf-1.cwl + id: workflow_any_input_with_file_provided + doc: Test Any parameter with file input to a workflow + tags: [ workflow, inline_javascript ] + +- job: tests/io-any-array.json + output: {"t1": [1, "moocow"]} + tool: tests/io-any-wf-1.cwl + id: workflow_any_input_with_mixed_array_provided + doc: Test Any parameter with array input to a workflow + tags: [ workflow, inline_javascript ] + +- job: tests/io-any-record.json + output: {"t1": {"moo": 1, "cow": 5}} + tool: tests/io-any-wf-1.cwl + id: workflow_any_input_with_record_provided + doc: Test Any parameter with record input to a tool + tags: [ workflow, inline_javascript ] + +- job: tests/empty.json + output: {"o": "the default value"} + tool: tests/io-union-input-default-wf.cwl + id: workflow_union_default_input_unspecified + doc: Test union type input to workflow with default unspecified + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/io-any-file.json + output: {"o": "File"} + tool: tests/io-union-input-default-wf.cwl + id: workflow_union_default_input_with_file_provided + doc: Test union type input to workflow with default specified as file + tags: [ workflow, inline_javascript, expression_tool ] + +- job: tests/empty.json + output: {"val": "moocow\n"} + tool: tests/step-valuefrom4-wf.cwl + id: workflowstep_valuefrom_string + doc: Test valueFrom on workflow step from literal (string). + tags: [ workflow, step_input ] + +- job: tests/wc-job.json + output: {"val1": "whale.txt\n", "val2": "step1_out\n"} + tool: tests/step-valuefrom5-wf.cwl + id: workflowstep_valuefrom_file_basename + doc: Test valueFrom on workflow step using basename. + tags: [ workflow, step_input ] + +- job: tests/output-arrays-int-job.json + output: {"o": [0, 1, 2]} + tool: tests/output-arrays-int.cwl + id: expression_tool_int_array_output + doc: Test output arrays in a tool (with ints). + tags: [ expression_tool, inline_javascript ] + +- job: tests/output-arrays-int-job.json + output: {"o": 12} + tool: tests/output-arrays-int-wf.cwl + id: workflowstep_int_array_input_output + doc: Test output arrays in a workflow (with ints). + tags: [ workflow, expression_tool, inline_javascript ] + +- job: tests/output-arrays-file-job.json + output: {"o": [ + { + "basename": "moo", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": Any, + "size": 0 + }, + { + "basename": "cow", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": Any, + "size": 0 + } + ]} + tool: tests/output-arrays-file-wf.cwl + id: workflow_file_array_output + doc: Test output arrays in a workflow (with Files). + tags: [ workflow, expression_tool, inline_javascript ] + +- job: "tests/empty.json" + output: {"cow": + { + "checksum": "sha1$7a788f56fa49ae0ba5ebde780efe4d6a89b5db47", + "basename": "cow", + "class": "File", + "size": 4, + "location": Any + }} + tool: "tests/docker-run-cmd.cwl" + id: docker_entrypoint + doc: Test Docker ENTRYPOINT usage + tags: [ command_line_tool, docker ] + +- job: "tests/job-input-array-one-empty-file.json" + output: {"output_file": + { + "checksum": "sha1$dad5a8472b87f6c5ef87d8fc6ef1458defc57250", + "basename": "output.txt", + "location": Any, + "class": "File", + "size": 11 + }} + tool: "tests/size-expression-tool.cwl" + id: clt_file_size_property_with_empty_file + doc: Test use of size in expressions for an empty file + tags: [ command_line_tool, inline_javascript ] + +- job: "tests/job-input-array-few-files.json" + output: {"output_file": + { + "checksum": "sha1$9def39730e8012bd09bf8387648982728501737d", + "basename": "output.txt", + "location": Any, + "class": "File", + "size": 31 + }} + tool: "tests/size-expression-tool.cwl" + id: clt_file_size_property_with_multi_file + doc: Test use of size in expressions for a few files + tags: [ command_line_tool, inline_javascript ] + +- job: tests/null-expression-echo-job.json + tool: tests/echo-tool.cwl + should_fail: true + id: any_without_defaults_unspecified_fails + doc: Test Any without defaults, unspecified, should fail. + tags: [ command_line_tool, required ] + +- job: tests/null-expression1-job.json + tool: tests/echo-tool.cwl + should_fail: true + id: any_without_defaults_specified_fails + doc: Test Any without defaults, specified, should fail. + tags: [ command_line_tool, required ] + +- job: tests/empty.json + output: + wc_output: + class: File + checksum: sha1$3596ea087bfdaf52380eae441077572ed289d657 + size: 3 + tool: tests/count-lines9-wf-noET.cwl + id: step_input_default_value_noexp + doc: Test default value on step input parameter, no ExpressionTool + tags: [ workflow, required ] + +- job: tests/cat-job.json + output: + wc_output: + class: File + checksum: sha1$e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e + size: 2 + tool: tests/count-lines11-wf-noET.cwl + id: step_input_default_value_overriden_noexp + doc: Test default value on step input parameter overridden by provided source, no ExpressionTool + tags: [ workflow, required ] + +- job: tests/wc-job.json + output: + wc_output: + class: File + checksum: sha1$3596ea087bfdaf52380eae441077572ed289d657 + size: 3 + id: nested_workflow_noexp + tool: tests/count-lines8-wf-noET.cwl + doc: Test nested workflow, without ExpressionTool + tags: [ workflow, subworkflow ] + +- job: tests/sum-job.json + output: + result: + class: File + checksum: sha1$ad552e6dc057d1d825bf49df79d6b98eba846ebe + size: 3 + id: wf_multiplesources_multipletypes_noexp + tool: tests/sum-wf-noET.cwl + doc: Test step input with multiple sources with multiple types, without ExpressionTool + tags: [ workflow, step_input, inline_javascript, multiple_input ] + +- tool: tests/dynresreq-workflow-tooldefault.cwl + job: tests/empty.json + output: + cores: + location: output + size: 2 + class: File + checksum: sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a + id: dynamic_resreq_wf_optional_file_default + doc: >- + Within a workflow, test accessing the size attribute of an optional input + File as part of a CommandLineTool's ResourceRequirement calculation. The + CommandLineTool input has a default value (a local file) and the workflow + nor the workflow step does not provide any value for this input. + tags: [ workflow, resource ] + +- tool: tests/dynresreq-workflow-stepdefault.cwl + job: tests/empty.json + output: + cores: + location: output + size: 2 + class: File + checksum: sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a + id: dynamic_resreq_wf_optional_file_step_default + doc: >- + Within a workflow, test accessing the size attribute of an optional input + File as part of a CommandLineTool's ResourceRequirement calculation. The + workflow step provides a default value (a local file) for this input and + the workflow itself does not provide any value for this input. + tags: [ workflow, resource ] + +- tool: tests/dynresreq-workflow-inputdefault.cwl + job: tests/empty.json + id: dynamic_resreq_wf_optional_file_wf_default + output: + cores: + location: output + size: 2 + class: File + checksum: sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a + doc: >- + Within a workflow, test accessing the size attribute of an optional input + File as part of a CommandLineTool's ResourceRequirement calculation. The + workflow itelf provides a default value (a local file) for this input. + tags: [ workflow, resource ] + +- job: tests/cat-job.json + output: {count_output: 1} + id: step_input_default_value_overriden_2nd_step + tool: tests/count-lines11-extra-step-wf.cwl + doc: >- + Test default value on step input parameter overridden by provided source. + With passthrough first step + tags: [ workflow, inline_javascript ] + +- job: tests/cat-job.json + output: + wc_output: + class: File + checksum: sha1$e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e + size: 2 + id: step_input_default_value_overriden_2nd_step_noexp + tool: tests/count-lines11-extra-step-wf-noET.cwl + doc: >- + Test default value on step input parameter overridden by provided source. + With passthrough first step and no ExpressionTool + tags: [ workflow, required ] + +- job: tests/empty.json + output: {count_output: 16} + id: step_input_default_value_overriden_2nd_step_null + tool: tests/count-lines11-null-step-wf.cwl + doc: >- + Test default value on step input parameter overridden by provided source. + With null producing first step + tags: [ workflow, inline_javascript ] + +- job: tests/empty.json + output: + wc_output: + class: File + checksum: sha1$3596ea087bfdaf52380eae441077572ed289d657 + size: 3 + id: step_input_default_value_overriden_2nd_step_null_noexp + tool: tests/count-lines11-null-step-wf-noET.cwl + doc: >- + Test default value on step input parameter overridden by provided source. + With null producing first step and no ExpressionTool + tags: [ workflow, required ] + +- tool: tests/cat-from-dir.cwl + job: tests/cat-from-dir-job.yaml + output: + output: + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + class: File + size: 13 + id: stdin_from_directory_literal_with_local_file + doc: Pipe to stdin from user provided local File via a Directory literal + tags: [ command_line_tool, required ] + +- tool: tests/cat-from-dir.cwl + job: tests/cat-from-dir-with-literal-file.yaml + output: + output: + checksum: sha1$ef88e689559565999700d6fea7cf7ba306d04360 + class: File + size: 26 + id: stdin_from_directory_literal_with_literal_file + doc: Pipe to stdin from literal File via a Directory literal + tags: [ command_line_tool, required ] + +- tool: tests/cat3-from-dir.cwl + job: tests/cat-from-dir-with-literal-file.yaml + output: + output_file: + checksum: sha1$ef88e689559565999700d6fea7cf7ba306d04360 + class: File + size: 26 + id: directory_literal_with_literal_file_nostdin + doc: Test non-stdin reference to literal File via a Directory literal + tags: [ command_line_tool, required ] + +- tool: tests/no-inputs-tool.cwl + output: + output: + checksum: sha1$1334e67fe9eb70db8ae14ccfa6cfb59e2cc24eae + location: output + class: File + size: 4 + id: no_inputs_commandlinetool + doc: Test CommandLineTool without inputs + tags: [ command_line_tool, required ] + +- tool: tests/no-outputs-tool.cwl + job: tests/cat-job.json + output: {} + id: no_outputs_commandlinetool + doc: Test CommandLineTool without outputs + tags: [ command_line_tool, required ] + +- tool: tests/no-inputs-wf.cwl + output: + output: + checksum: sha1$1334e67fe9eb70db8ae14ccfa6cfb59e2cc24eae + location: output + class: File + size: 4 + id: no_inputs_workflow + doc: Test Workflow without inputs + tags: [ workflow, required ] + +- tool: tests/no-outputs-wf.cwl + job: tests/cat-job.json + output: {} + id: no_outputs_workflow + doc: Test Workflow without outputs + tags: [ workflow, required ] + +- tool: tests/anon_enum_inside_array.cwl + job: tests/anon_enum_inside_array.yml + output: + result: + class: File + checksum: "sha1$2132943d72c39423e0b9425cbc40dfd5bf3e9cb2" + size: 39 + id: anonymous_enum_in_array + doc: Test an anonymous enum inside an array inside a record + tags: [command_line_tool, required] + +- tool: tests/anon_enum_inside_array_inside_schemadef.cwl + job: tests/anon_enum_inside_array_inside_schemadef.yml + output: + result: + class: File + checksum: "sha1$f17c7d81f66e1520fca25b96b90eeeae5bbf08b0" + size: 39 + id: schema-def_anonymous_enum_in_array + doc: Test an anonymous enum inside an array inside a record, SchemaDefRequirement + tags: [command_line_tool, schema_def] + +- job: tests/wc-job.json + output: + output: + class: File + checksum: sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376 + location: output + size: 1111 + tool: tests/cat-tool-shortcut.cwl + id: stdin_shorcut + doc: Test command execution in with stdin and stdout redirection using stdin shortcut + tags: [ command_line_tool, docker ] + +- job: tests/record-secondaryFiles-job.yml + output: {} + tool: tests/record-in-secondaryFiles.cwl + id: secondary_files_in_unnamed_records + doc: Test secondaryFiles on anonymous record fields + tags: [ command_line_tool, required ] + +- job: tests/record-secondaryFiles-job.yml + output: {} + tool: tests/record-sd-secondaryFiles.cwl + id: secondary_files_in_named_records + doc: Test secondaryFiles on SchemaDefRequirement record fields + tags: [ command_line_tool, schema_def ] + +- job: tests/empty.json + output: { + "record_output": { + "f1": { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "A", + "secondaryFiles": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "A.s2", + "class": "File", + "size": 0 + } + ], + "class": "File", + "size": 0 + }, + "f2": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "B", + "secondaryFiles": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "B.s3", + "class": "File", + "size": 0 + } + ], + "class": "File", + "size": 0 + }, + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "C", + "secondaryFiles": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "C.s3", + "class": "File", + "size": 0 + } + ], + "class": "File", + "size": 0 + } + ] + } + } + tool: tests/record-out-secondaryFiles.cwl + id: secondary_files_in_output_records + doc: Test secondaryFiles on output record fields + tags: [ command_line_tool, required ] + +- job: tests/record-secondaryFiles-job.yml + output: {} + tool: tests/record-in-secondaryFiles-wf.cwl + id: secondary_files_workflow_propagation + doc: Test secondaryFiles propagation through workflow + tags: [ workflow, required ] + +- job: tests/record-secondaryFiles-job.yml + should_fail: true + tool: tests/record-in-secondaryFiles-missing-wf.cwl + id: secondary_files_missing + doc: Test checking when secondaryFiles are missing + tags: [ workflow, required ] + +- job: tests/record-format-job.yml + output: {} + tool: tests/record-in-format.cwl + id: input_records_file_entry_with_format + doc: Test format on anonymous record fields + tags: [ command_line_tool, required ] + +- job: tests/record-format-job2.yml + should_fail: true + tool: tests/record-in-format.cwl + id: input_records_file_entry_with_format_and_bad_regular_input_file_format + doc: Test file format checking on parameter + tags: [ command_line_tool, format_checking ] + +- job: tests/record-format-job3.yml + should_fail: true + tool: tests/record-in-format.cwl + doc: Test file format checking on record field + id: input_records_file_entry_with_format_and_bad_entry_file_format + tags: [ command_line_tool, format_checking ] + +- job: tests/record-format-job4.yml + should_fail: true + tool: tests/record-in-format.cwl + doc: Test file format checking on array item + id: input_records_file_entry_with_format_and_bad_entry_array_file_format + tags: [ command_line_tool, format_checking ] + +- job: tests/record-secondaryFiles-job.yml + output: { + "f1out": { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "A", + "size": 0, + "format": "http://example.com/format1" + }, + "record_output": { + "f2out": { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "B", + "size": 0, + "format": "http://example.com/format2" + } + } + } + tool: tests/record-out-format.cwl + id: record_output_file_entry_format + doc: Test format on output record fields + tags: [ command_line_tool, format_checking ] + +- job: tests/wf-loadContents-job.yml + output: { + "my_int": 42 + } + tool: tests/wf-loadContents.cwl + id: workflow_input_inputBinding_loadContents + doc: Test WorkflowInputParameter.inputBinding.loadContents + tags: [ workflow, step_input, inline_javascript, expression_tool ] + +- job: tests/wf-loadContents-job.yml + output: { + "my_int": 42 + } + tool: tests/wf-loadContents2.cwl + doc: Test WorkflowInputParameter.loadContents + id: workflow_input_loadContents_without_inputBinding + tags: [ workflow, step_input, inline_javascript, expression_tool ] + +- job: tests/wf-loadContents-job.yml + output: { + "my_int": 42 + } + tool: tests/wf-loadContents3.cwl + id: expression_tool_input_loadContents + doc: Test loadContents on InputParameter.loadContents (expression) + tags: [ workflow, step_input, inline_javascript, expression_tool ] + +- job: tests/wf-loadContents-job.yml + output: { + "my_int": 42 + } + tool: tests/wf-loadContents4.cwl + id: workflow_step_in_loadContents + doc: Test WorkflowStepInput.loadContents + tags: [ workflow, step_input, inline_javascript, expression_tool ] + +- job: tests/empty.json + should_fail: true + tool: tests/timelimit.cwl + id: timelimit_basic + doc: Test that job fails when exceeding time limit + tags: [ command_line_tool, timelimit, work_reuse ] + +- job: tests/empty.json + should_fail: true + tool: tests/timelimit2.cwl + id: timelimit_invalid + doc: Test invalid time limit value + tags: [ command_line_tool, timelimit, json_schema_invalid ] + +- job: tests/empty.json + output: {} + tool: tests/timelimit3.cwl + id: timelimit_zero_unlimited + doc: Test zero timelimit means no limit + tags: [ command_line_tool, timelimit, work_reuse ] + +- job: tests/empty.json + should_fail: true + tool: tests/timelimit4.cwl + id: timelimit_from_expression + doc: Test expression in time limit + tags: [ command_line_tool, timelimit, inline_javascript, work_reuse ] + +- job: tests/empty.json + output: + status: Done + tool: tests/timelimit5.cwl + id: timelimit_expressiontool + doc: Test timelimit in expressiontool is ignored + tags: [ expression_tool, timelimit, inline_javascript ] + +- job: tests/empty.json + should_fail: true + tool: tests/timelimit-wf.cwl + id: timelimit_basic_wf + doc: Test that tool in workflow fails when exceeding time limit + tags: [ workflow, timelimit ] + +- job: tests/empty.json + output: + o: time passed + tool: tests/timelimit2-wf.cwl + id: timelimit_invalid_wf + doc: Test that workflow level time limit is not applied to workflow execution time + tags: [ workflow, timelimit, inline_javascript ] + +- job: tests/empty.json + output: + o: time passed + tool: tests/timelimit3-wf.cwl + id: timelimit_zero_unlimited_wf + doc: Test zero timelimit means no limit in workflow + tags: [ workflow, timelimit, inline_javascript, work_reuse ] + +- job: tests/empty.json + should_fail: true + tool: tests/timelimit4-wf.cwl + id: timelimit_from_expression_wf + doc: Test expression in time limit in workflow + tags: [ workflow, timelimit, inline_javascript, work_reuse ] + +- job: tests/empty.json + output: {} + tool: tests/networkaccess.cwl + doc: Test networkaccess enabled + id: networkaccess + tags: [ command_line_tool, networkaccess ] + +- job: tests/empty.json + should_fail: true + tool: tests/networkaccess2.cwl + id: networkaccess_disabled + doc: Test networkaccess is disabled by default + tags: [ networkaccess, command_line_tool ] + +- job: tests/stage-array-job.json + tool: tests/stage-array.cwl + doc: Test null and array input in InitialWorkDirRequirement + output: + "output": { + "basename": "lsout", + "checksum": "sha1$e453d26efd859a9abc80ae1a1d9d63db72376053", + "class": "File", + "location": "lsout", + "size": 32 + } + tags: [ resource, command_line_tool, initial_work_dir ] + id: initial_work_dir_for_null_and_arrays + +- job: tests/stage-array-dirs-job.yml + tool: tests/stage-array-dirs.cwl + doc: Test array of directories InitialWorkDirRequirement + output: { + "output": [ + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "a", + "size": 0 + }, + { + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "B", + "size": 0 + } + ] + } + tags: [ resource, command_line_tool, initial_work_dir ] + id: initial_work_dir_for_array_dirs + +- job: tests/env-job3.yaml + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-tool3.cwl + id: cwl_requirements_addition + doc: Test requirements in input document via EnvVarRequirement + tags: [ command_line_tool, input_object_requirements ] + +- job: tests/env-job3.yaml + output: + out: + class: File + checksum: sha1$b3ec4ed1749c207e52b3a6d08c59f31d83bff519 + location: out + size: 15 + tool: tests/env-tool4.cwl + id: cwl_requirements_override_expression + doc: Test conflicting requirements in input document via EnvVarRequirement and expression + tags: [ command_line_tool, input_object_requirements ] + +- job: tests/env-job4.yaml + output: + out: + class: File + checksum: sha1$715e62184492851512a020c36ab7118eca114a59 + location: out + size: 23 + id: cwl_requirements_override_static + tool: tests/env-tool3.cwl + doc: Test conflicting requirements in input document via EnvVarRequirement + tags: [ command_line_tool, input_object_requirements ] + +- job: tests/initialworkdirrequirement-docker-out-job.json + output: + OUTPUT: + "checksum": "sha1$aeb3d11bdf536511649129f4077d5cda6a324118" + "location": "ref.fasta" + "secondaryFiles": [{ + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "location": "ref.fasta.fai", + "class": "File", + "size": 0 + }] + "class": "File" + "size": 12010 + tool: tests/initialworkdir-glob-fullpath.cwl + id: initial_workdir_output_glob + doc: Test full path glob output of InitialWorkDir + tags: [ initial_work_dir, command_line_tool ] + +- job: tests/empty.json + should_fail: true + tool: tests/glob-path-error.cwl + id: glob_outside_outputs_fails + doc: Test fail trying to glob outside output directory + tags: [ command_line_tool, docker ] + +- job: tests/empty.json + tool: tests/symlink-illegal.cwl + doc: symlink to file outside of working directory should NOT be retrieved + output: + output_file: + "class": "File" + "size": 27 + "baesname": "symlink.txt" + "checksum": "sha1$cd28ec34f3f9425aca544b6332453708e8aaa82a" + should_fail: true + tags: [ command_line_tool ] + id: illegal_symlink + +- job: tests/empty.json + tool: tests/symlink-legal.cwl + doc: symlink to file inside of working directory should be retrieved + output: + output_file: + "class": "File" + "size": 27 + "basename": "symlink.txt" + "checksum": "sha1$cd28ec34f3f9425aca544b6332453708e8aaa82a" + tags: [ command_line_tool ] + id: legal_symlink + +- job: tests/empty.json + tool: tests/inp_update_wf.cwl + doc: inplace update has side effect on file content + tags: [ inplace_update, workflow ] + output: + a: 4 + b: 4 + id: modify_file_content + +- job: tests/empty.json + tool: tests/inpdir_update_wf.cwl + doc: inplace update has side effect on directory content + tags: [ inplace_update, workflow ] + output: { + a: [ + { + "basename": "blurb", + "class": "File", + "location": "blurb" + } + ], + b: [ + { + "basename": "blurb", + "class": "File", + "location": "blurb" + } + ] + } + id: modify_directory_content + +- doc: Test that OutputBinding.glob accepts Directories + job: tests/empty.json + id: outputbinding_glob_directory + tool: tests/glob_directory.cwl + output: + directories: + - basename: "a_dir" + class: "Directory" + listing: [] + - basename: "b_dir" + class: "Directory" + listing: [] + - basename: "c_dir" + class: "Directory" + listing: [] + tags: [ required, command_line_tool ] + +- doc: Test that array of input files can be staged to directory with entryname + id: stage_file_array + tool: tests/stage_file_array.cwl + job: tests/stage_file_array.job.json + output: + output: + [ + { + "basename": "sfa-1.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-1.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + }, + { + "basename": "sfa-2.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-2.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + } + ] + tags: [ command_line_tool, initial_work_dir, inline_javascript ] + +- doc: Test that array of input files can be staged to directory with basename + id: stage_file_array_basename + tool: tests/stage_file_array_basename.cwl + job: tests/stage_file_array.job.json + output: + output: + [ + { + "basename": "sfa-1.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-1.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + }, + { + "basename": "sfa-2.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-2.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + } + ] + tags: [ command_line_tool, initial_work_dir, inline_javascript ] + +- doc: Test that if array of input files are staged to directory with basename and entryname, entryname overrides + id: stage_file_array_entryname_overrides + tool: tests/stage_file_array_basename_and_entryname.cwl + job: tests/stage_file_array.job.json + output: + output: + [ + { + "basename": "sfa-1.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-1.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + }, + { + "basename": "sfa-2.txt", + "checksum": "sha1$4c1cd0638ab3580310823fd1556d27ecb4816df6", + "class": "File", + "size": 49 + }, + { + "basename": "sfa-2.txt.sec", + "checksum": "sha1$40f4ee1bcd1a9466fcd2e48cf7fc3798025d2f9a", + "class": "File", + "size": 59 + } + ] + tags: [ command_line_tool, initial_work_dir, inline_javascript ] + +- job: tests/empty.json + output: + foo: + class: File + basename: foo + checksum: sha1$fa98d6085770a79e44853d575cd3ab40c0f1f4de + tool: tests/runtime-paths-distinct.cwl + id: tmpdir_is_not_outdir + doc: Test that runtime.tmpdir is not runtime.outdir + tags: [ command_line_tool ] + +- job: tests/listing-job.yml + tool: tests/listing_none1.cwl + id: listing_default_none + output: + out: true + doc: Test that default behavior is 'no_listing' if not specified + tags: [ command_line_tool ] + +- job: tests/listing-job.yml + tool: tests/listing_none2.cwl + id: listing_requirement_none + output: + out: true + doc: Test that 'listing' is not present when LoadListingRequirement is 'no_listing' + tags: [ command_line_tool, load_listing ] + +- job: tests/listing-job.yml + tool: tests/listing_none3.cwl + id: listing_loadListing_none + output: + out: true + doc: Test that 'listing' is not present when loadListing on input parameter is 'no_listing' + tags: [ command_line_tool ] + +- job: tests/listing-job.yml + tool: tests/listing_shallow1.cwl + id: listing_requirement_shallow + output: + out: true + doc: > + Test that 'listing' is present in top directory object but not + subdirectory object when LoadListingRequirement is + 'shallow_listing' + tags: [ command_line_tool, load_listing ] + +- job: tests/listing-job.yml + tool: tests/listing_shallow2.cwl + id: listing_loadListing_shallow + output: + out: true + doc: > + Test that 'listing' is present in top directory object but not + subdirectory object when loadListing on input parameter loadListing is + 'shallow_listing' + tags: [ command_line_tool ] + +- job: tests/listing-job.yml + tool: tests/listing_shallow3.cwl + id: listing_outputBinding_loadListing + output: + out: { + "basename": "file2", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "location": "file2", + "size": 0 + } + doc: > + Test that loadListing works when used on outputBinding. + tags: [ command_line_tool ] + +- job: tests/listing-job.yml + tool: tests/listing_deep1.cwl + id: listing_requirement_deep + output: + out: true + doc: > + Test that 'listing' is present in top directory object and in + subdirectory objects when LoadListingRequirement is + 'deep_listing' + tags: [ command_line_tool, load_listing ] + +- job: tests/listing-job.yml + tool: tests/listing_deep2.cwl + id: listing_loadListing_deep + output: + out: true + doc: > + Test that 'listing' is present in top directory object and in + subdirectory objects when input parameter loadListing is + 'deep_listing' + tags: [ command_line_tool ] + +- job: tests/echo-position-expr-job.yml + tool: tests/echo-position-expr.cwl + id: inputBinding_position_expr + output: + out: "🕺 1 singular sensation!\n" + doc: > + Test for expression in the InputBinding.position field; also test using emoji in CWL document and tool output + tags: [ command_line_tool, required ] + +- job: tests/empty.json + tool: tests/exitcode.cwl + id: outputEval_exitCode + output: + code: 7 + doc: Can access exit code in outputEval + tags: [ command_line_tool, required ] + +- tool: tests/echo-tool-packed.cwl + job: tests/env-job.json + output: + {"out": "hello test env\n"} + id: any_input_param_graph_no_default + doc: Test use of $graph without specifying which process to run + tags: [ required, command_line_tool ] + +- tool: tests/echo-tool-packed2.cwl + job: tests/env-job.json + output: + {"out": "hello test env\n"} + id: any_input_param_graph_no_default_hashmain + doc: > + Test use of $graph without specifying which process to run, + hash-prefixed "main" + tags: [ required, command_line_tool ] + +- tool: tests/optional-numerical-output-0.cwl + job: tests/empty.json + output: + {"out": 0} + id: optional_numerical_output_returns_0_not_null + doc: | + Test that optional number output is returned as 0, not null + tags: [ inline_javascript, command_line_tool ] + +- job: tests/empty.json + tool: tests/cores_float.cwl + id: cores_float + doc: Test float value for coresMin(Max) is rounded up in runtime.cores + output: + output: { + "location": "cores.txt", + "size": 2, + "class": "File", + "checksum": "sha1$7448d8798a4380162d4b56f9b452e2f6f9e24e7a" + } + tags: [ resource, command_line_tool ] + +- job: tests/empty.json + tool: tests/storage_float.cwl + id: storage_float + doc: Test float value for ram/tmpdir/outdir Min(Max) is rounded up + output: + output: { + "location": "values.txt", + "basename": "values.txt", + "class": "File", + "checksum": "sha1$c73f68407c144c5336a6c14e7ec79ee470231bd7", + "size": 12 + } + tags: [ resource, command_line_tool ] + +- job: tests/synth-file-job.yml + tool: tests/synth-file.cwl + id: cat_synthetic_file + doc: Test the generation of a synthetic file with inline contents. + output: + sequence: { + "class": "File", + "checksum": "sha1$5d64b71392b1e00a3ad893db02d381d58262c2d6", + "size": 7 + } + tags: [ required, command_line_tool ] + +- $import: tests/string-interpolation/test-index.yaml + +- $import: tests/conditionals/test-index.yaml +- $import: tests/secondaryfiles/test-index.yaml +- $import: tests/mixed-versions/test-index.yaml +- $import: tests/loadContents/test-index.yaml +- $import: tests/iwd/test-index.yaml +- $import: tests/scatter/test-index.yaml + +- job: tests/empty.json + tool: tests/params_broken_null.cwl + id: params_broken_null + doc: Test parameter reference that refers to null.something + should_fail: true + tags: [ required, command_line_tool ] + +- job: tests/empty.json + tool: tests/params_broken_length_of_non_list.cwl + doc: Test parameter reference that refers to length of non-array input + should_fail: true + tags: [ required, command_line_tool ] + id: length_for_non_array + +- job: tests/length_non_array_input.yml + tool: tests/params_input_length_non_array.cwl + doc: Test 'length' in parameter reference where it does not refer to length of an array input + tags: [ required, command_line_tool ] + output: + output1: 1 + output2: 2 + output3: 3 + id: user_defined_length_in_parameter_reference + +- tool: tests/cat4-from-dir.cwl + job: tests/cat-from-dir-with-literal-file-in-subdir.yaml + output: + output_file: + checksum: sha1$ef88e689559565999700d6fea7cf7ba306d04360 + class: File + size: 26 + id: directory_literal_with_literal_file_in_subdir_nostdin + doc: Test non-stdin reference to literal File via a nested Directory literal + tags: [ command_line_tool, required ] + +- id: colon_in_paths + output: + log: + class: File + basename: re:sult + checksum: sha1$d7d6491030bfa0ce17bab3a648e603f2a55bf503 + size: 14 + result: + class: Directory + basename: A:Gln2Cys_result + listing: + - basename: A:Gln2Cys + checksum: sha1$2928c9c6fa02098aee8c31bf44099f3bf8c91013 + class: File + size: 18 + job: tests/colon:test:job.yaml + tool: tests/colon:test.cwl + doc: Confirm that colons are tolerated in input paths, string values, stdout shortcut, and output file & directory names + tags: [ required, command_line_tool ] + +- id: colon_in_output_path + output: + result: + class: Directory + basename: A:Gln2Cys_result + listing: + - basename: hello.txt + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + class: File + size: 13 + job: tests/colon_test_output_job.yaml + tool: tests/colon_test_output.cwl + doc: Confirm that colons are tolerated in output directory names + tags: [ required, command_line_tool ] + +- id: record_with_default + output: + same_record: + first: y + second: 23 + third: 2.3 + fourth: 4242424242 + fifth: 4200000000000000000000000000000000000000000 + sixth: + class: File + basename: whale.txt + size: 1111 + checksum: sha1$327fc7aedf4f6b69a42a7c8b808dc5a7aff61376 + tool: tests/paramref_arguments_roundtrip.cwl + doc: Confirm that records with defaults are accepted + tags: [ required, command_line_tool ] + +- id: record_outputeval + output: + references: + genome_fa: + class: File + basename: GRCm38.primary_assembly.genome.fa + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + size: 0 + annotation_gtf: + class: File + basename: gencode.vM21.primary_assembly.annotation.gtf + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + size: 0 + tool: tests/record_outputeval.cwl + doc: Use of outputEval on a record itself, not the fields of the record + tags: [ inline_javascript, command_line_tool ] + +- id: record_outputeval_nojs + output: + references: + annotation_gtf: gencode.vM21.primary_assembly.annotation.gtf + genome_fa: GRCm38.primary_assembly.genome.fa + summary: + class: File + checksum: sha1$c0bf6c4db73d4fc08ee24c7288825c09f9cb3cea + size: 79 + tool: tests/record_outputeval_nojs.cwl + doc: Use of outputEval on a record itself, not the fields of the record (without javascript) + tags: [ required, command_line_tool ] + +- id: staging-basename + output: {} + tool: tests/staging-basename/wf_ren.cwl + doc: | + Use of expression tool to change basename of file, then correctly staging + the file using the new name. + tags: [ workflow, inline_javascript, expression_tool ] + +- id: runtime-outdir + output: { + "stuff": { + "class": "Directory", + "listing": [ + { + "basename": "baz.txt", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "size": 0 + }, + { + "basename": "foo", + "class": "Directory", + "listing": [ + { + "basename": "bar.txt", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "size": 0 + } + ] + } + ] + } + } + tool: tests/runtime-outdir.cwl + doc: | + Use of $(runtime.outdir) for outputBinding glob. + tags: [ required, command_line_tool ] + +- tool: tests/record-order.cwl + job: tests/record-order-job.json + output: + args: [-a, -b, '1', -c, '3', -d, -e, '2', -f, '4'] + id: record_order_with_input_bindings + doc: Test sorting arguments at each level (inputBinding for all levels) + tags: [ command_line_tool, required ] + +- id: output_reference_workflow_input + output: { + "last": "me" + } + tool: tests/output_reference_workflow_input.cwl + doc: | + Direct use of Workflow level input fields in the outputs. + tags: [ required, workflow ] + +- id: stdout_chained_commands + output: { + "out": "a\nb\n" + } + tool: tests/stdout_chained_commands.cwl + doc: | + Test that chaining two echo calls causes the workflow tool to emit the output to stdout. + This is to confirm that the workflow tool will **not** create an expression such as + `echo a && echo b > out.txt`, but instead will produce the correct `echo a && echo b`, + and capture the output correctly. + tags: [ shell_command, command_line_tool ] + +- id: multiple-input-feature-requirement + output: { + "hello_world_in_two_lines": [ + "hello\n", + "world\n" + ] + } + tool: tests/multiple_input_feature_requirement.cwl + doc: | + MultipleInputFeatureRequirement on workflow outputs. + tags: [ workflow, multiple_input ] + +- id: js-input-record + # The output does not have the last \n due to the -n passed to echo + output: { + "out": "JS\nwith\nrecord" + } + tool: tests/js-input-record.cwl + job: tests/js-input-record.json + doc: | + A test case for JS with an input record. + tags: [ inline_javascript, command_line_tool ] + +- tool: tests/schemadef_types_with_import-wf.cwl + job: tests/schemadef_types_with_import-job.json + output: { + "out": "1970-01-01T00:00:00Z\n" + } + id: schemadef_types_with_import + doc: >- + Test SchemaDefRequirement with a workflow, with the `$import` under types. + It is similar to schemadef-wf, but the `$import` is different. + tags: [ workflow, schema_def ] + +- id: filename_with_hash_mark + tool: tests/cat-tool.cwl + job: tests/octo.yml + doc: Test for correct handling of URL-quoting in filename to refer to filename containing a hash mark + output: { + "output": { + "basename": "output", + "checksum": "sha1$06b0c59808c236447d065db8f7d2a60de0a805bf", + "class": "File", + "location": "output", + "size": 8 + } + } + tags: [ required, command_line_tool ] + +- id: capture_files + tool: tests/capture-files.cwl + job: tests/dir-job.yml + doc: Test that a type error is raised if directories are returned by glob evaluation when type is File + should_fail: true + tags: [ required, command_line_tool ] + +- id: capture_dirs + tool: tests/capture-dirs.cwl + job: tests/dir-job.yml + doc: Test that a type error is raised if files are returned by glob evaluation when type is Directory + should_fail: true + tags: [ required, command_line_tool ] + +- id: capture_files_and_dirs + tool: tests/capture-files-and-dirs.cwl + job: tests/dir-job.yml + doc: Test that both files and directories are captured by glob evaluation when type is [Directory, File] + output: { + "result": [ + { + "basename": "a", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "size": 0 + }, + { + "basename": "b", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "size": 0 + }, + { + "basename": "c", + "class": "Directory", + "listing": [ + { + "basename": "d", + "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", + "class": "File", + "size": 0 + } + ] + } + ] + } + tags: [ required, command_line_tool ] + +- id: very_big_and_very_floats + doc: Confirm that very big and very small numbers are represented using decimals, not scientific notation + tool: tests/floats_small_and_large.cwl + job: tests/empty.json + output: { "result": "0.00001 0.0000123 123000 1230000" } + tags: [ inline_javascript, command_line_tool ] + +- id: very_big_and_very_floats_nojs + doc: Confirm that very big and very small numbers are represented using decimals, not scientific notation + tool: tests/floats_small_and_large_nojs.cwl + job: tests/empty.json + output: { + "result": { + "basename": "dump", + "class": "File", + "checksum": "sha1$8a3913a553b8f29d47b99c1f4b0f6c2ee833cdc2", + "size": 32 + } + } + tags: [ required, command_line_tool ] + +- id: default_with_falsey_value + job: tests/empty.json + tool: tests/default_with_falsey_value.cwl + doc: Confirms that "false"-like (but not 'null') values override any default + output: + out: + class: File + basename: file.txt + size: 14 + checksum: sha1$0074b1841ab5103cc0442f739d9fb41a33b602ee + tags: [ step_input, workflow ] + +- id: nested_types + job: tests/nested_types.yaml + tool: tests/nested_types.cwl + doc: demonstrate usage of nested types + output: + their_name: "Foo Bar" + tags: [ require, command_line_tool ] + +- id: paramref_arguments_runtime + job: tests/empty.json + tool: tests/paramref_arguments_runtime.cwl + doc: confirm that $runtime is available to parameter references in arguments + tags: [ required, command_line_tool ] + output: + runtime: + cores: Any + outdir: Any + outdirSize: Any + ram: Any + tmpdir: Any + tmpdirSize: Any + +- id: paramref_arguments_self + tool: tests/paramref_arguments_self.cwl + job: tests/empty.json + doc: confirm that $self is available and set to null in arguments + tags: [ required, command_line_tool ] + output: + self: null + +- id: paramref_arguments_inputs + job: tests/empty.json + tool: tests/paramref_arguments_inputs.cwl + doc: confirm that $inputs is available to parameter references in arguments + tags: [ required, command_line_tool ] + output: + result: + a_double: 1000000000000000000000000000000000000000000 + a_float: 4.2 + a_long: 4147483647 + a_string: z + a_string_array: [ a, b, c ] + a_true: true + an_array_of_doubles: [ 1000000000000000000000000000000000000000000, -1000000000000000000000000000000000000000000 ] + an_array_of_falses: [ false, false, false ] + an_array_of_floats: [ 2.3, 4.2 ] + an_array_of_ints: [ 42, 23 ] + an_array_of_longs: [ 4147483647, -4147483647 ] + an_array_of_mixed_booleans: [ false, true, false ] + an_array_of_trues: [ true, true, true ] + an_int: 42 + +- $import: tests/loop/test-index.yaml diff --git a/cwltool/schemas/v1.3.0-dev1/contrib.md b/cwltool/schemas/v1.3.0-dev1/contrib.md new file mode 100644 index 000000000..6dbc4e4c4 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/contrib.md @@ -0,0 +1,9 @@ +Authors: + +This specification is a draft, and has not yet been approved as an official CWL standard. + +Contributors to v1.3.0-dev1: + +Incorporates the work of past authors and contributors to [CWL v1.0](https://www.commonwl.org/v1.0/Workflow.html), + [CWL v1.1](https://www.commonwl.org/v1.1/Workflow.html), +and [CWL v1.2](https://www.commonwl.org/v1.2/Workflow.html). diff --git a/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl b/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl new file mode 100644 index 000000000..bc2296257 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/cwl-runner.cwl @@ -0,0 +1,59 @@ +#!/usr/bin/env cwl-runner +class: CommandLineTool +cwlVersion: v1.2 + +doc: | + Generic interface to run a Common Workflow Language tool or workflow from the + command line. To be implemented by each CWL compliant execution platform for + testing conformance to the standard and optionally for use by users. + +inputs: + outdir: + type: string? + doc: | + Output directory, defaults to the current directory + inputBinding: + prefix: "--outdir" + + quiet: + type: boolean? + doc: no diagnostic output + inputBinding: + prefix: "--quiet" + + version: + type: boolean? + doc: report the name & version, then quit without further processing + inputBinding: + prefix: "--version" + + processfile: + type: File? + doc: | + The CommandLineTool, ExpressionTool, or Workflow description to run. + Optional if the jobfile has a `cwl:tool` field to indicate which process + description to run. + inputBinding: + position: 1 + + jobfile: + type: File + doc: The input job document. + inputBinding: + position: 2 + +baseCommand: cwl-runner + +outputs: + log: stderr + output_object_document: + type: stdout # in the CWL Output Object Document format + format: iana:application/json + +successCodes: + - 0 # success + +permanentFailCodes: + - 33 # failure due to unimplemented feature + +$namespaces: { iana: https://www.iana.org/assignments/media-types/ } diff --git a/cwltool/schemas/v1.3.0-dev1/index.md b/cwltool/schemas/v1.3.0-dev1/index.md new file mode 100644 index 000000000..9a719947a --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/index.md @@ -0,0 +1,79 @@ +# Common Workflow Language Standards, v1.3.0-dev1 + +This is an un-official, non-ratified, development version of the future CWL v1.3 +standard. Any changes from v1.2 could be modified before final ratification. + +The [User Guide](http://www.commonwl.org/user_guide/) provides a gentle +introduction to writing CWL command line tools and workflows. + +The CWL [Command Line Tool Description Standard](CommandLineTool.html) +specifies the document schema and execution semantics for wrapping and +executing command line tools. + +The CWL [Workflow Description Standard](Workflow.html) specifies the document +schema and execution semantics for composing workflows from components such as +command line tools and other workflows. + +The +[Semantic Annotations for Linked Avro Data (SALAD) Specification](SchemaSalad.html) +specifies the preprocessing steps that must be applied when loading CWL +documents and the schema language used to write the above specifications. + +Also available are inheritance graphs (as SVG images) for the [Schema Salad object model](salad.svg) and the [CWL object model](cwl.svg). + +# Running the CWL conformance tests + +Install a CWL runner of your choice. The reference runner can be installed as +the default runner by doing: +``` +pip install cwlref-runner +``` + +Install the CWL test parser: +``` +pip install cwltest +``` +You may need to activate a virtualenv first, or do a local install by adding `--user` after `install` above. + +From within a copy of [this repository](https://github.com/common-workflow-language/cwl-v1.2) (e.g. cwl-v1.2) execute the main test script +``` +./run_test.sh +``` + +If the CWL runner isn't installed as `cwl-runner` then you can specify the name for the runner: +``` +./run_test.sh RUNNER=cwltool +``` + +You can also specify additional options that are specific for the particular CWL runner you are using. +For example, with CWL reference runner you can turn on parallel execution mode: +``` +./run_test.sh RUNNER=cwltool EXTRA=--parallel +``` + +This can be combined with launching more than one CWL conformance test at once with `-j`: +``` +./run_test.sh -j4 RUNNER=cwltool EXTRA=--parallel +``` + +You can list all the tests +``` +./run_test.sh -l +``` + +You can run a particular test +``` +./run_test.sh -n23 +``` + + +If you are running tests using the CWL reference runner "cwltool" for an unreleased version of the CWL standards use the `--enable-dev` flag: +``` +./run_test.sh RUNNER=cwltool EXTRA=--enable-dev +``` + + +For details of options you can pass to the test script, do: +``` +./run_test.sh --help +``` diff --git a/cwltool/schemas/v1.3.0-dev1/intro.md b/cwltool/schemas/v1.3.0-dev1/intro.md new file mode 100644 index 000000000..21873f3bc --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/intro.md @@ -0,0 +1,21 @@ +# Status of this document + +This document is the product of the [Common Workflow Language working +group](https://www.commonwl.org/). The +source for the latest version of this document is available at + +https://github.com/common-workflow-language/cwl-v1.2/ + +The products of the CWL working group (including this document) are made available +under the terms of the Apache License, version 2.0. + + + +# Introduction + +The Common Workflow Language (CWL) working group is an informal, multi-vendor +working group consisting of various organizations and individuals that have an +interest in portability of data analysis workflows. The goal is to create +specifications like this one that enable data scientists to describe analysis +tools and workflows that are powerful, easy to use, portable, and support +reproducibility. diff --git a/cwltool/schemas/v1.3.0-dev1/invocation.md b/cwltool/schemas/v1.3.0-dev1/invocation.md new file mode 100644 index 000000000..8f7e8d732 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/invocation.md @@ -0,0 +1,184 @@ +# Running a Command + +To accommodate the enormous variety in syntax and semantics for input, runtime +environment, invocation, and output of arbitrary programs, a CommandLineTool +defines an "input binding" that describes how to translate abstract input +parameters to a concrete program invocation, and an "output binding" that +describes how to generate output parameters from program output. + +## Input binding + +The tool command line is built by applying command line bindings to the +input object. Bindings are listed either as part of an [input +parameter](#CommandInputParameter) using the `inputBinding` field, or +separately using the `arguments` field of the CommandLineTool. + +The algorithm to build the command line is as follows. In this algorithm, +the sort key is a list consisting of one or more numeric or string +elements. Strings are sorted lexicographically based on UTF-8 encoding. + + 1. Collect `CommandLineBinding` objects from `arguments`. Assign a sorting + key `[position, i]` where `position` is + [`CommandLineBinding.position`](#CommandLineBinding) and `i` + is the index in the `arguments` list. + + 2. Collect `CommandLineBinding` objects from the `inputs` schema and + associate them with values from the input object. Where the input type + is a record, array, or map, recursively walk the schema and input object, + collecting nested `CommandLineBinding` objects and associating them with + values from the input object. + + 3. Create a sorting key by taking the value of the `position` field at + each level leading to each leaf binding object. If `position` is not + specified, it is not added to the sorting key. For bindings on arrays + and maps, the sorting key must include the array index or map key + following the position. If and only if two bindings have the same sort + key, the tie must be broken using the ordering of the field or parameter + name immediately containing the leaf binding. + + 4. Sort elements using the assigned sorting keys. Numeric entries sort + before strings. + + 5. In the sorted order, apply the rules defined in + [`CommandLineBinding`](#CommandLineBinding) to convert bindings to actual + command line elements. + + 6. Insert elements from `baseCommand` at the beginning of the command + line. + +## Runtime environment + +All files listed in the input object must be made available in the runtime +environment. The implementation may use a shared or distributed file +system or transfer files via explicit download to the host. Implementations +may choose not to provide access to files not explicitly specified in the input +object or process requirements. + +Output files produced by tool execution must be written to the +**designated output directory**. The initial current working +directory when executing the tool must be the designated output +directory. The designated output directory should be empty, except +for files or directories specified using +[InitialWorkDirRequirement](#InitialWorkDirRequirement). + +Files may also be written to the **designated temporary directory**. This +directory must be isolated and not shared with other processes. Any files +written to the designated temporary directory may be automatically deleted by +the workflow platform immediately after the tool terminates. + +For compatibility, files may be written to the **system temporary directory** +which must be located at `/tmp`. Because the system temporary directory may be +shared with other processes on the system, files placed in the system temporary +directory are not guaranteed to be deleted automatically. A tool +must not use the system temporary directory as a back-channel communication with +other tools. It is valid for the system temporary directory to be the same as +the designated temporary directory. + +When executing the tool, the tool must execute in a new, empty environment +with only the environment variables described below; the child process must +not inherit environment variables from the parent process except as +specified or at user option. + + * `HOME` must be set to the designated output directory. + * `TMPDIR` must be set to the designated temporary directory. + * `PATH` may be inherited from the parent process, except when run in a + container that provides its own `PATH`. + * Variables defined by [EnvVarRequirement](#EnvVarRequirement) + * The default environment of the container, such as when using + [DockerRequirement](#DockerRequirement) + +An implementation may forbid the tool from writing to any location in the +runtime environment file system other than the designated temporary directory, +system temporary directory, and designated output directory. An implementation +may provide read-only input files, and disallow in-place update of input files. +The designated temporary directory, system temporary directory and designated +output directory may each reside on different mount points on different file +systems. + +An implementation may forbid the tool from directly accessing network +resources. Correct tools must not assume any network access unless they have +the 'networkAccess' field of a ['NetworkAccess'](#NetworkAccess) requirement set +to `true` but even then this does not imply a publicly routable IP address or +the ability to accept inbound connections. + +The `runtime` section available in [parameter references](#Parameter_references) +and [expressions](#Expressions) contains the following fields. As noted +earlier, an implementation may perform deferred resolution of runtime fields by providing +[opaque strings](#opaque-strings) for any or all of the following fields; parameter +references and expressions may only use the literal string value of the field and must +not perform computation on the contents. + + * `runtime.outdir`: an absolute path to the designated output directory + * `runtime.tmpdir`: an absolute path to the designated temporary directory + * `runtime.cores`: number of CPU cores reserved for the tool process + * `runtime.ram`: amount of RAM in mebibytes (2\*\*20) reserved for the tool process + * `runtime.outdirSize`: reserved storage space available in the designated output directory + * `runtime.tmpdirSize`: reserved storage space available in the designated temporary directory + +For `cores`, `ram`, `outdirSize` and `tmpdirSize`, if an implementation can't +provide the actual number of reserved resources during the expression evaluation time, +it should report back the minimal requested amount. + +See [ResourceRequirement](#ResourceRequirement) for details on how to +describe the hardware resources required by a tool. + +The standard input stream, the standard output stream, and/or the standard error +stream may be redirected as described in the [`stdin`](#stdin), +[`stdout`](#stdout), and [`stderr`](#stderr) fields. + +## Execution + +Once the command line is built and the runtime environment is created, the +actual tool is executed. + +The standard error stream and standard output stream may be captured by +platform logging facilities for storage and reporting. If there are multiple +commands logically chained (e.g. `echo a && echo b`) implementations must +capture the output of all the commands, and not only the output of the last +command (i.e. the following is incorrect `echo a && echo b > captured`, +as the output of `echo a` is not included in `captured`). + +Tools may be multithreaded or spawn child processes; however, when the +parent process exits, the tool is considered finished regardless of whether +any detached child processes are still running. Tools must not require any +kind of console, GUI, or web based user interaction in order to start and +run to completion. + +The exit code of the process indicates if the process completed +successfully. By convention, an exit code of zero is treated as success +and non-zero exit codes are treated as failure. This may be customized by +providing the fields `successCodes`, `temporaryFailCodes`, and +`permanentFailCodes`. An implementation may choose to default unspecified +non-zero exit codes to either `temporaryFailure` or `permanentFailure`. + +The exit code of the process is available to expressions in +`outputEval` as `runtime.exitCode`. + +## Output binding + +If the output directory contains a file named "cwl.output.json", that +file must be loaded and used as the output object. In this case, the +output object should still be type-checked against the `outputs` +section, but `outputBinding` is ignored. + +For Files and Directories, if the value of `path` is a relative path +pattern (does not begin with a slash '/') then it is resolved relative +to the output directory. If the value of the "path" is an absolute +path pattern (it does begin with a slash '/') then it must refer to a +path within the output directory. It is an error for "path" to refer +outside the output directory. + +Similarly, if a File or Directory "cwl.output.json" contains +`location`, it is resolved as relative reference IRI with a base IRI +representing the output directory. If `location` contains some other +absolute IRI with a scheme supported by the implementation, the +implementation may choose to accept it. + +If both `path` and `location` are provided on a File or Directory in +"cwl.output.json", `path` takes precedence. + +If there is no "cwl.output.json", the output object must be generated +by walking the parameters listed in `outputs` and applying output +bindings to the tool output. Output bindings are associated with +output parameters using the `outputBinding` field. See +[`CommandOutputBinding`](#CommandOutputBinding) for details. diff --git a/cwltool/schemas/v1.3.0-dev1/salad/LICENSE.txt b/cwltool/schemas/v1.3.0-dev1/salad/LICENSE.txt new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/cwltool/schemas/v1.3.0-dev1/salad/README.rst b/cwltool/schemas/v1.3.0-dev1/salad/README.rst new file mode 100644 index 000000000..4f27a88b5 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/README.rst @@ -0,0 +1,263 @@ +|Linux Build Status| |Code coverage| |Documentation Status| |CII Best Practices| + +.. |Linux Build Status| image:: https://github.com/common-workflow-language/schema_salad/actions/workflows/ci-tests.yml/badge.svg?branch=main + :target: https://github.com/common-workflow-language/schema_salad/actions/workflows/ci-tests.yml +.. |Code coverage| image:: https://codecov.io/gh/common-workflow-language/schema_salad/branch/main/graph/badge.svg + :target: https://codecov.io/gh/common-workflow-language/schema_salad +.. |Documentation Status| image:: https://readthedocs.org/projects/schema-salad/badge/?version=latest + :target: https://schema-salad.readthedocs.io/en/latest/?badge=latest + :alt: Documentation Status +.. |CII Best Practices| image:: https://bestpractices.coreinfrastructure.org/projects/1867/badge + :target: https://bestpractices.coreinfrastructure.org/projects/1867 + +Schema Salad +------------ + +Salad is a schema language for describing JSON or YAML structured +linked data documents. Salad schema describes rules for +preprocessing, structural validation, and hyperlink checking for +documents described by a Salad schema. Salad supports rich data +modeling with inheritance, template specialization, object +identifiers, object references, documentation generation, code +generation, and transformation to RDF_. Salad provides a bridge +between document and record oriented data modeling and the Semantic +Web. + +The Schema Salad library is Python 3.8+ only. + +Installation +------------ + +:: + + pip3 install schema_salad + +If you intend to use the `schema-salad-tool --codegen=python` feature, please +include the `[pycodegen]` extra:: + + pip3 install schema_salad[pycodegen] + +To install from source:: + + git clone https://github.com/common-workflow-language/schema_salad + cd schema_salad + pip3 install . + # or pip3 install .[pycodegen] if needed + +Commands +-------- + +Schema salad can be used as a command line tool or imported as a Python module:: + + $ schema-salad-tool + usage: schema-salad-tool [-h] [--rdf-serializer RDF_SERIALIZER] [--skip-schemas] + [--strict-foreign-properties] [--print-jsonld-context] + [--print-rdfs] [--print-avro] [--print-rdf] [--print-pre] + [--print-index] [--print-metadata] [--print-inheritance-dot] + [--print-fieldrefs-dot] [--codegen language] [--codegen-target CODEGEN_TARGET] + [--codegen-examples directory] [--codegen-package dotted.package] + [--codegen-copyright copyright_string] [--print-oneline] + [--print-doc] [--strict | --non-strict] + [--verbose | --quiet | --debug] [--only ONLY] [--redirect REDIRECT] + [--brand BRAND] [--brandlink BRANDLINK] [--brandstyle BRANDSTYLE] + [--brandinverse] [--primtype PRIMTYPE] [--version] + [schema] [document] + + $ python + >>> import schema_salad + +Validate a schema:: + + $ schema-salad-tool myschema.yml + +Validate a document using a schema:: + + $ schema-salad-tool myschema.yml mydocument.yml + +Generate HTML documentation:: + + $ schema-salad-tool --print-doc myschema.yml > myschema.html + $ # or + $ schema-salad-doc myschema.yml > myschema.html + +Get JSON-LD context:: + + $ schema-salad-tool --print-jsonld-context myschema.yml mydocument.yml + +Convert a document to JSON-LD:: + + $ schema-salad-tool --print-pre myschema.yml mydocument.yml > mydocument.jsonld + +Generate Python classes for loading/generating documents described by the schema +(Requires the `[pycodegen]` extra):: + + $ schema-salad-tool --codegen=python myschema.yml > myschema.py + +Display inheritance relationship between classes as a graphviz 'dot' file and +render as SVG:: + + $ schema-salad-tool --print-inheritance-dot myschema.yml | dot -Tsvg > myschema.svg + +Codegen Examples +---------------- + +The examples in the tables below are helpful to see how to use the output of `schema-salad-tool --codegen` +in different languages for loading and/or creating/editing/saving objects. + +First set of examples is using the `CWL v1.2 schema `_: + ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Language | Repository | Serialization Example | Deserialization Example | ++=============+=========================================================+======================================================================================================================================================+============================================================================================================================================================================+ +| Python | https://github.com/common-workflow-language/cwl-utils/ | `create_cwl_from_objects.py `_ | `load_document() `_ | ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Java | https://github.com/common-workflow-language/cwljava/ | (Not yet implemented) | `PackedWorkflowClassTest.java `_ | ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| TypeScript | https://github.com/common-workflow-lab/cwl-ts-auto | `Creating, editing, and saving CWL docs with TypeScript `_ | `Loading CWL documents with TypeScript `_ | ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| .Net | https://github.com/common-workflow-lab/CWLDotNet | `Creating, editing, and saving CWL docs with .Net `_ | `Loading CWL documents with .Net `_ | ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| C++ | https://github.com/common-workflow-lab/cwl-cpp-auto | `cwl_output_example.cpp `_ | `cwl_input_example.cpp `_ | ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| D | https://github.com/common-workflow-lab/cwl-d-auto | `How to use `_ | `How to use `_ | ++-------------+---------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Second set of examples is for the `Galaxy Workflow Format 2 `_ schema: + ++-------------+------------------------------------------------------------------------------------+ +| Language | Path | ++=============+====================================================================================+ +| Python | https://github.com/galaxyproject/gxformat2/blob/master/gxformat2/schema/v19_09.py | ++-------------+------------------------------------------------------------------------------------+ +| Java | https://github.com/galaxyproject/gxformat2/tree/master/java | ++-------------+------------------------------------------------------------------------------------+ +| TypeScript | https://github.com/galaxyproject/gxformat2/tree/master/typescript | ++-------------+------------------------------------------------------------------------------------+ + +Quick Start +----------- + +Let's say you have a 'basket' record that can contain items measured either by +weight or by count. Here's an example:: + + basket: + - product: bananas + price: 0.39 + per: pound + weight: 1 + - product: cucumbers + price: 0.79 + per: item + count: 3 + +We want to validate that all the expected fields are present, the +measurement is known, and that "count" cannot be a fractional value. +Here is an example schema to do that:: + + - name: Product + doc: | + The base type for a product. This is an abstract type, so it + can't be used directly, but can be used to define other types. + type: record + abstract: true + fields: + product: string + price: float + + - name: ByWeight + doc: | + A product, sold by weight. Products may be sold by pound or by + kilogram. Weights may be fractional. + type: record + extends: Product + fields: + per: + type: + type: enum + symbols: + - pound + - kilogram + jsonldPredicate: '#per' + weight: float + + - name: ByCount + doc: | + A product, sold by count. The count must be a integer value. + type: record + extends: Product + fields: + per: + type: + type: enum + symbols: + - item + jsonldPredicate: '#per' + count: int + + - name: Basket + doc: | + A basket of products. The 'documentRoot' field indicates it is a + valid starting point for a document. The 'basket' field will + validate subtypes of 'Product' (ByWeight and ByCount). + type: record + documentRoot: true + fields: + basket: + type: + type: array + items: Product + +You can check the schema and document in schema_salad/tests/basket_schema.yml +and schema_salad/tests/basket.yml:: + + $ schema-salad-tool basket_schema.yml basket.yml + Document `basket.yml` is valid + + +Documentation +------------- + +See the specification_ and the metaschema_ (salad schema for itself). For an +example application of Schema Salad see the Common Workflow Language_. + + +Rationale +--------- + +The JSON data model is an popular way to represent structured data. It is +attractive because of it's relative simplicity and is a natural fit with the +standard types of many programming languages. However, this simplicity comes +at the cost that basic JSON lacks expressive features useful for working with +complex data structures and document formats, such as schemas, object +references, and namespaces. + +JSON-LD is a W3C standard providing a way to describe how to interpret a JSON +document as Linked Data by means of a "context". JSON-LD provides a powerful +solution for representing object references and namespaces in JSON based on +standard web URIs, but is not itself a schema language. Without a schema +providing a well defined structure, it is difficult to process an arbitrary +JSON-LD document as idiomatic JSON because there are many ways to express the +same data that are logically equivalent but structurally distinct. + +Several schema languages exist for describing and validating JSON data, such as +JSON Schema and Apache Avro data serialization system, however none +understand linked data. As a result, to fully take advantage of JSON-LD to +build the next generation of linked data applications, one must maintain +separate JSON schema, JSON-LD context, RDF schema, and human documentation, +despite significant overlap of content and obvious need for these documents to +stay synchronized. + +Schema Salad is designed to address this gap. It provides a schema language +and processing rules for describing structured JSON content permitting URI +resolution and strict document validation. The schema language supports linked +data through annotations that describe the linked data interpretation of the +content, enables generation of JSON-LD context and RDF schema, and production +of RDF triples by applying the JSON-LD context. The schema language also +provides for robust support of inline documentation. + +.. _JSON-LD: http://json-ld.org +.. _Avro: http://avro.apache.org +.. _metaschema: https://github.com/common-workflow-language/schema_salad/blob/main/schema_salad/metaschema/metaschema.yml +.. _specification: http://www.commonwl.org/v1.2/SchemaSalad.html +.. _Language: https://github.com/common-workflow-language/cwl-v1.2/blob/v1.2.0/CommandLineTool.yml +.. _RDF: https://www.w3.org/RDF/ diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name.yml new file mode 100644 index 000000000..60b947827 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name.yml @@ -0,0 +1,46 @@ +- | + ## Field name resolution + + The document schema declares the vocabulary of known field names. During + preprocessing traversal, field name in the document which are not part of + the schema vocabulary must be resolved to absolute URIs. Under "strict" + validation, it is an error for a document to include fields which are not + part of the vocabulary and not resolvable to absolute URIs. Fields names + which are not part of the vocabulary are resolved using the following + rules: + + * If a field name URI begins with a namespace prefix declared in the + document context (`@context`) followed by a colon `:`, the prefix and + colon must be replaced by the namespace declared in `@context`. + + * If there is a vocabulary term which maps to the URI of a resolved + field, the field name must be replace with the vocabulary term. + + * If a field name URI is an absolute URI consisting of a scheme and path + and is not part of the vocabulary, no processing occurs. + + Field name resolution is not relative. It must not be affected by the + base URI. + + ### Field name resolution example + + Given the following schema: + + ``` +- $include: field_name_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: field_name_src.yml +- | + ``` + + This becomes: + + ``` +- $include: field_name_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_proc.yml new file mode 100644 index 000000000..a53ef4bbd --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_proc.yml @@ -0,0 +1,8 @@ + { + "base": "one", + "form": { + "base": "two", + "http://example.com/three": "three", + }, + "http://example.com/acid#four": "four" + } diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_schema.yml new file mode 100644 index 000000000..a49423868 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_schema.yml @@ -0,0 +1,15 @@ +{ + "$namespaces": { + "acid": "http://example.com/acid#" + }, + "$graph": [{ + "name": "ExampleType", + "type": "record", + "documentRoot": true, + "fields": [{ + "name": "base", + "type": "string", + "jsonldPredicate": "http://example.com/base" + }] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_src.yml new file mode 100644 index 000000000..1ed79b967 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/field_name_src.yml @@ -0,0 +1,8 @@ + { + "base": "one", + "form": { + "http://example.com/base": "two", + "http://example.com/three": "three", + }, + "acid:four": "four" + } diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res.yml new file mode 100644 index 000000000..3955f8926 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res.yml @@ -0,0 +1,70 @@ +- | + ## Identifier resolution + + The schema may designate one or more fields as identifier fields to identify + specific objects. Processing must resolve relative identifiers to absolute + identifiers using the following rules: + + * If an identifier URI begins with `#` it is a current document + fragment identifier. It is resolved relative to the base URI by + setting or replacing the fragment portion of the base URI. + + * If an identifier URI contains `#` in some other position it is a + relative URI with fragment identifier. It is resolved relative + to the base URI by stripping the last path segment from the base + URI and adding the identifier followed by the fragment. + + * If an identifier URI does not contain a scheme and does not + contain `#` it is a parent relative fragment identifier. + + * If an identifier URI is a parent relative fragment identifier + and the base URI does not contain a document fragment, set the + document fragment on the base URI. + + * If an identifier URI is a parent relative fragment identifier + and the object containing this identifier is assigned to a + parent object field defined with `subscope` in + `jsonldPredicate`, append a slash `/` to the base URI fragment + followed by the value of the parent field `subscope`. Then + append the identifier as described in the next rule. + + * If an identifier URI is a parent relative fragment identifier + and the base URI contains a document fragment, append a slash + `/` to the fragment followed by the identifier field to the + fragment portion of the base URI. + + * If an identifier URI begins with a namespace prefix declared in + `$namespaces` followed by a colon `:`, the prefix and colon must be + replaced by the namespace declared in `$namespaces`. + + * If an identifier URI is an absolute URI consisting of a scheme and path, + no processing occurs. + + When preprocessing visits a node containing an identifier, that identifier + must be used as the base URI to process child nodes. + + It is an error for more than one object in a document to have the same + absolute URI. + + ### Identifier resolution example + + Given the following schema: + + ``` +- $include: ident_res_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: ident_res_src.yml +- | + ``` + + This becomes: + + ``` +- $include: ident_res_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_proc.yml new file mode 100644 index 000000000..db9d82202 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_proc.yml @@ -0,0 +1,25 @@ +{ + "id": "http://example.com/base", + "form": { + "id": "http://example.com/base#one", + "things": [ + { + "id": "http://example.com/base#one/two" + }, + { + "id": "http://example.com/base#three" + }, + { + "id": "http://example.com/four#five", + }, + { + "id": "http://example.com/acid#six", + }, + { + "subscopeField": { + "id": "http://example.com/base#one/thisIsASubscope/seven" + } + } + ], + } +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_schema.yml new file mode 100644 index 000000000..9e4a28b6b --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_schema.yml @@ -0,0 +1,24 @@ +{ + "$namespaces": { + "acid": "http://example.com/acid#" + }, + "$graph": [{ + "name": "ExampleType", + "type": "record", + "documentRoot": true, + "fields": [{ + "name": "id", + "type": "string", + "jsonldPredicate": "@id" + }]}, { + "name": "SubscopeType", + "type": "record", + "fields": [{ + "name": "subscopeField", + "type": "ExampleType", + "jsonldPredicate": { + "subscope": "thisIsASubscope" + } + }] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_src.yml new file mode 100644 index 000000000..4b634ff41 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/ident_res_src.yml @@ -0,0 +1,25 @@ + { + "id": "http://example.com/base", + "form": { + "id": "one", + "things": [ + { + "id": "two" + }, + { + "id": "#three", + }, + { + "id": "four#five", + }, + { + "id": "acid:six", + }, + { + "subscopeField": { + "id": "seven" + } + } + ], + } + } diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/import_include.md b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/import_include.md new file mode 100644 index 000000000..980240e94 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/import_include.md @@ -0,0 +1,145 @@ +## Import + +During preprocessing traversal, an implementation must resolve `$import` +directives. An `$import` directive is an object consisting of exactly one +field `$import` specifying resource by URI string. It is an error if there +are additional fields in the `$import` object, such additional fields must +be ignored. + +The URI string must be resolved to an absolute URI using the link +resolution rules described previously. Implementations must support +loading from `file`, `http` and `https` resources. The URI referenced by +`$import` must be loaded and recursively preprocessed as a Salad document. +The external imported document does not inherit the context of the +importing document, and the default base URI for processing the imported +document must be the URI used to retrieve the imported document. If the +`$import` URI includes a document fragment, the fragment must be excluded +from the base URI used to preprocess the imported document. + +If the `$import` node is in an array and the import operation yields an array, +it is flattened to the parent array. Otherwise the `$import` node is replaced +in the document structure by the object or array yielded from the import operation. + +URIs may reference document fragments which refer to specific an object in +the target document. This indicates that the `$import` node must be +replaced by only the object with the appropriate fragment identifier. + +It is a fatal error if an import directive refers to an external resource +or resource fragment which does not exist or is not accessible. + +### Import example: replacing the `$import` node + +import.json: +``` +{ + "hello": "world" +} + +``` + +parent.json: +``` +{ + "form": { + "bar": { + "$import": "import.json" + } + } +} + +``` + +This becomes: + +``` +{ + "form": { + "bar": { + "hello": "world" + } + } +} +``` + +### Import example: flattening the `$import`ed array + +import.json: +``` +[ "hello", "world" ] +``` + +parent.json: +``` +{ + "form": [ + "bar", + { + "$import": "import.json" + } + ] +} + +``` + +This becomes: + +``` +{ + "form": [ + "bar", + "hello", + "world" + ] +} +``` + +## Include + +During preprocessing traversal, an implementation must resolve `$include` +directives. An `$include` directive is an object consisting of exactly one +field `$include` specifying a URI string. It is an error if there are +additional fields in the `$include` object, such additional fields must be +ignored. + +The URI string must be resolved to an absolute URI using the link +resolution rules described previously. The URI referenced by `$include` must +be loaded as a text data. Implementations must support loading from +`file`, `http` and `https` resources. Implementations may transcode the +character encoding of the text data to match that of the parent document, +but must not interpret or parse the text document in any other way. + +Once loaded, the `$include` node is replaced in the document structure by a +string containing the text data loaded from the resource. + +It is a fatal error if an import directive refers to an external resource +which does not exist or is not accessible. + +### Include example + +parent.json: +``` +{ + "form": { + "bar": { + "$include": "include.txt" + } + } +} + +``` + +include.txt: +``` +hello world + +``` + +This becomes: + +``` +{ + "form": { + "bar": "hello world" + } +} +``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res.yml new file mode 100644 index 000000000..4ce36fb49 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res.yml @@ -0,0 +1,55 @@ +- | + ## Link resolution + + The schema may designate one or more fields as link fields that reference other + objects. Processing must resolve links to absolute URIs using the + following rules: + + * If a reference URI is prefixed with `#` it is a relative + fragment identifier. It is resolved relative to the base URI by setting + or replacing the fragment portion of the base URI. + + * If a reference URI does not contain a scheme and is not prefixed with `#` + it is a path relative reference. If the reference URI contains `#` in any + position other than the first character, the reference URI must be divided + into a path portion and a fragment portion split on the first instance of + `#`. The path portion is resolved relative to the base URI by the following + rule: if the path portion of the base URI ends in a slash `/`, append the + path portion of the reference URI to the path portion of the base URI. If + the path portion of the base URI does not end in a slash, replace the final + path segment with the path portion of the reference URI. Replace the + fragment portion of the base URI with the fragment portion of the reference + URI. + + * If a reference URI begins with a namespace prefix declared in `$namespaces` + followed by a colon `:`, the prefix and colon must be replaced by the + namespace declared in `$namespaces`. + + * If a reference URI is an absolute URI consisting of a scheme and path, + no processing occurs. + + Link resolution must not affect the base URI used to resolve identifiers + and other links. + + ### Link resolution example + + Given the following schema: + + ``` +- $include: link_res_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: link_res_src.yml +- | + ``` + + This becomes: + + ``` +- $include: link_res_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_proc.yml new file mode 100644 index 000000000..03e539d5e --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_proc.yml @@ -0,0 +1,21 @@ +{ + "$base": "http://example.com/base", + "link": "http://example.com/base/zero", + "form": { + "link": "http://example.com/one", + "things": [ + { + "link": "http://example.com/two" + }, + { + "link": "http://example.com/base#three" + }, + { + "link": "http://example.com/four#five", + }, + { + "link": "http://example.com/acid#six", + } + ] + } +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_schema.yml new file mode 100644 index 000000000..76420d3b5 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_schema.yml @@ -0,0 +1,16 @@ +{ + "$namespaces": { + "acid": "http://example.com/acid#" + }, + "$graph": [{ + "name": "ExampleType", + "type": "record", + "fields": [{ + "name": "link", + "type": "string", + "jsonldPredicate": { + "_type": "@id" + } + }] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_src.yml new file mode 100644 index 000000000..23f7a295e --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/link_res_src.yml @@ -0,0 +1,21 @@ +{ + "$base": "http://example.com/base", + "link": "http://example.com/base/zero", + "form": { + "link": "one", + "things": [ + { + "link": "two" + }, + { + "link": "#three", + }, + { + "link": "four#five", + }, + { + "link": "acid:six", + } + ] + } +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res.yml new file mode 100644 index 000000000..636cc71f9 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res.yml @@ -0,0 +1,36 @@ +- | + ## Identifier maps + + The schema may designate certain fields as having a `mapSubject`. If the + value of the field is a JSON object, it must be transformed into an array of + JSON objects. Each key-value pair from the source JSON object is a list + item, each list item must be a JSON object, and the value of the key is + assigned to the field specified by `mapSubject`. + + Fields which have `mapSubject` specified may also supply a `mapPredicate`. + If the value of a map item is not a JSON object, the item is transformed to a + JSON object with the key assigned to the field specified by `mapSubject` and + the value assigned to the field specified by `mapPredicate`. + + ### Identifier map example + + Given the following schema: + + ``` +- $include: map_res_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: map_res_src.yml +- | + ``` + + This becomes: + + ``` +- $include: map_res_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_proc.yml new file mode 100644 index 000000000..704c7bbb6 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_proc.yml @@ -0,0 +1,12 @@ +{ + "mapped": [ + { + "value": "daphne", + "key": "fred" + }, + { + "value": "scooby", + "key": "shaggy" + } + ] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_schema.yml new file mode 100644 index 000000000..086cc29ba --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_schema.yml @@ -0,0 +1,30 @@ +{ + "$graph": [{ + "name": "MappedType", + "type": "record", + "documentRoot": true, + "fields": [{ + "name": "mapped", + "type": { + "type": "array", + "items": "ExampleRecord" + }, + "jsonldPredicate": { + "mapSubject": "key", + "mapPredicate": "value" + } + }], + }, + { + "name": "ExampleRecord", + "type": "record", + "fields": [{ + "name": "key", + "type": "string" + }, { + "name": "value", + "type": "string" + } + ] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_src.yml new file mode 100644 index 000000000..e6e3e0bd2 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/map_res_src.yml @@ -0,0 +1,8 @@ +{ + "mapped": { + "shaggy": { + "value": "scooby" + }, + "fred": "daphne" + } +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/metaschema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/metaschema.yml new file mode 100644 index 000000000..f696e0aed --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/metaschema.yml @@ -0,0 +1,405 @@ +$base: "https://w3id.org/cwl/salad#" + +$namespaces: + sld: "https://w3id.org/cwl/salad#" + dct: "http://purl.org/dc/terms/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + rdfs: "http://www.w3.org/2000/01/rdf-schema#" + xsd: "http://www.w3.org/2001/XMLSchema#" + +$graph: + +- name: "Semantic_Annotations_for_Linked_Avro_Data" + type: documentation + doc: + - $include: salad.md + - $import: field_name.yml + - $import: ident_res.yml + - $import: link_res.yml + - $import: vocab_res.yml + - $include: import_include.md + - $import: map_res.yml + - $import: typedsl_res.yml + - $import: sfdsl_res.yml + +- name: "Link_Validation" + type: documentation + doc: | + # Link validation + + Once a document has been preprocessed, an implementation may validate + links. The link validation traversal may visit fields which the schema + designates as link fields and check that each URI references an existing + object in the current document, an imported document, file system, or + network resource. Failure to validate links may be a fatal error. Link + validation behavior for individual fields may be modified by `identity` and + `noLinkCheck` in the `jsonldPredicate` section of the field schema. + + +- name: "Schema_Validation" + type: documentation + doc: | + # Validating a document against a schema + + To validate a document against the schema, first [apply + preprocessing](#Document_preprocessing), then, use the following + algorithm. + + 1. The document root must be an object or a list. If the document root is an + object containing the field `$graph` (which must be a list of + objects), then validation applies to each object in the list. + 2. For each object, attempt to validate as one of the record types + flagged with `documentRoot: true`. + 3. To validate a record, go through `fields` and recursively + validate each field of the object. + 4. For fields with a list of types (type union), go through each + type in the list and recursively validate the type. For the + field to be valid, at least one type in the union must be valid. + 5. Missing fields are considered `null`. To validate, the allowed types + for the field must include `null` + 6. Primitive types are null, boolean, int, long, float, double, + string. To validate, the value in the document must have one + of these type. For numerics, the value appearing in the + document must fit into the specified type. + 7. To validate an array, the value in the document must be a list, + and each item in the list must recursively validate as a type + in `items`. + 8. To validate an enum, the value in the document be a string, and + the value must be equal to the short name of one of the values + listed in `symbols`. + 9. As a special case, a field with the `Expression` type validates string values + which contain a CWL parameter reference or expression in the form + `$(...)` or `${...}` + +# - name: "JSON_LD_Context" +# type: documentation +# doc: | +# # Generating JSON-LD Context + +# How to generate the json-ld context... + + +- $import: metaschema_base.yml + +- name: JsonldPredicate + type: record + doc: | + Attached to a record field to define how the parent record field is handled for + URI resolution and JSON-LD context generation. + fields: + - name: _id + type: string? + jsonldPredicate: + _id: sld:_id + _type: "@id" + identity: true + doc: | + The predicate URI that this field corresponds to. + Corresponds to JSON-LD `@id` directive. + - name: _type + type: string? + doc: | + The context type hint, corresponds to JSON-LD `@type` directive. + + * If the value of this field is `@id` and `identity` is false or + unspecified, the parent field must be resolved using the link + resolution rules. If `identity` is true, the parent field must be + resolved using the identifier expansion rules. + + * If the value of this field is `@vocab`, the parent field must be + resolved using the vocabulary resolution rules. + + - name: _container + type: string? + doc: | + Structure hint, corresponds to JSON-LD `@container` directive. + - name: identity + type: boolean? + doc: | + If true and `_type` is `@id` this indicates that the parent field must + be resolved according to identity resolution rules instead of link + resolution rules. In addition, the field value is considered an + assertion that the linked value exists; absence of an object in the loaded document + with the URI is not an error. + - name: noLinkCheck + type: boolean? + doc: | + If true, this indicates that link validation traversal must stop at + this field. This field (it is is a URI) or any fields under it (if it + is an object or array) are not subject to link checking. + - name: mapSubject + type: string? + doc: | + If the value of the field is a JSON object, it must be transformed + into an array of JSON objects, where each key-value pair from the + source JSON object is a list item, the list items must be JSON objects, + and the key is assigned to the field specified by `mapSubject`. + - name: mapPredicate + type: string? + doc: | + Only applies if `mapSubject` is also provided. If the value of the + field is a JSON object, it is transformed as described in `mapSubject`, + with the addition that when the value of a map item is not an object, + the item is transformed to a JSON object with the key assigned to the + field specified by `mapSubject` and the value assigned to the field + specified by `mapPredicate`. + - name: refScope + type: int? + doc: | + If the field contains a relative reference, it must be resolved by + searching for valid document references in each successive parent scope + in the document fragment. For example, a reference of `foo` in the + context `#foo/bar/baz` will first check for the existence of + `#foo/bar/baz/foo`, followed by `#foo/bar/foo`, then `#foo/foo` and + then finally `#foo`. The first valid URI in the search order shall be + used as the fully resolved value of the identifier. The value of the + refScope field is the specified number of levels from the containing + identifier scope before starting the search, so if `refScope: 2` then + "baz" and "bar" must be stripped to get the base `#foo` and search + `#foo/foo` and the `#foo`. The last scope searched must be the top + level scope before determining if the identifier cannot be resolved. + - name: typeDSL + type: boolean? + doc: | + Field must be expanded based on the the Schema Salad type DSL. + - name: secondaryFilesDSL + type: boolean? + doc: | + Field must be expanded based on the the Schema Salad secondary file DSL. + - name: subscope + type: string? + doc: | + Append the subscope to the current scope when performing + identifier resolution to objects under this field. + +- name: SpecializeDef + type: record + fields: + - name: specializeFrom + type: string + doc: "The data type to be replaced" + jsonldPredicate: + _id: "sld:specializeFrom" + _type: "@id" + refScope: 1 + + - name: specializeTo + type: string + doc: "The new data type to replace with" + jsonldPredicate: + _id: "sld:specializeTo" + _type: "@id" + refScope: 1 + + +- name: NamedType + type: record + abstract: true + docParent: "#Schema" + fields: + - name: name + type: string + jsonldPredicate: "@id" + doc: "The identifier for this type" + - name: inVocab + type: boolean? + default: true + doc: | + If "true" (the default), include the short name of this type + in the vocabulary. The vocabulary are all the symbols (field + names and other identifiers, such as classes and enum values) + which can be used in the document without a namespace prefix. + These are the keys of the JSON-LD context. If false, do not + include the short name in the vocabulary. + + This is useful for specifying schema extensions that will be + included in validation without introducing ambiguity by + introducing non-standard terms into the vocabulary. + + +- name: DocType + type: record + extends: Documented + abstract: true + docParent: "#Schema" + fields: + - name: docParent + type: string? + doc: | + Hint to indicate that during documentation generation, documentation + for this type should appear in a subsection under `docParent`. + jsonldPredicate: + _id: "sld:docParent" + _type: "@id" + + - name: docChild + type: + - string? + - string[]? + doc: | + Hint to indicate that during documentation generation, documentation + for `docChild` should appear in a subsection under this type. + jsonldPredicate: + _id: "sld:docChild" + _type: "@id" + + - name: docAfter + type: string? + doc: | + Hint to indicate that during documentation generation, documentation + for this type should appear after the `docAfter` section at the same + level. + jsonldPredicate: + _id: "sld:docAfter" + _type: "@id" + + +- name: SchemaDefinedType + type: record + extends: DocType + doc: | + Abstract base for schema-defined types. + abstract: true + fields: + - name: jsonldPredicate + type: + - string? + - JsonldPredicate? + doc: | + Annotate this type with linked data context. + jsonldPredicate: sld:jsonldPredicate + + - name: documentRoot + type: boolean? + doc: | + If true, indicates that the type is a valid at the document root. At + least one type in a schema must be tagged with `documentRoot: true`. + jsonldPredicate: sld:documentRoot + + +- name: SaladRecordField + type: record + extends: RecordField + doc: "A field of a record." + fields: + - name: jsonldPredicate + type: + - string? + - JsonldPredicate? + doc: | + Annotate this type with linked data context. + jsonldPredicate: "sld:jsonldPredicate" + - name: default + type: Any? + jsonldPredicate: + _id: sld:default + noLinkCheck: true + doc: | + The default value to use for this field if the field is missing or "null". + + +- name: SaladRecordSchema + docParent: "#Schema" + type: record + extends: [NamedType, RecordSchema, SchemaDefinedType] + documentRoot: true + specialize: + RecordField: SaladRecordField + fields: + - name: abstract + type: boolean? + doc: | + If true, this record is abstract and may be used as a base for other + records, but is not valid on its own. Inherited fields may be + re-specified to narrow their type. + + - name: extends + type: + - string? + - string[]? + jsonldPredicate: + _id: "sld:extends" + _type: "@id" + refScope: 1 + doc: | + Indicates that this record inherits fields from one or more base records. + Inherited fields may be re-specified to narrow their type. + - name: specialize + type: + - SpecializeDef[]? + doc: | + Only applies if `extends` is declared. Apply type specialization using the + base record as a template. For each field inherited from the base + record, replace any instance of the type `specializeFrom` with + `specializeTo`. + jsonldPredicate: + _id: "sld:specialize" + mapSubject: specializeFrom + mapPredicate: specializeTo + +- name: SaladEnumSchema + docParent: "#Schema" + type: record + extends: [NamedType, EnumSchema, SchemaDefinedType] + documentRoot: true + doc: | + Define an enumerated type. + fields: + - name: extends + type: + - string? + - string[]? + jsonldPredicate: + _id: "sld:extends" + _type: "@id" + refScope: 1 + doc: | + Indicates that this enum inherits symbols from a base enum. + + +- name: SaladMapSchema + docParent: "#Schema" + type: record + extends: [NamedType, MapSchema, SchemaDefinedType] + documentRoot: true + doc: | + Define a map type. + + +- name: SaladUnionSchema + docParent: "#Schema" + type: record + extends: [NamedType, UnionSchema, DocType] + documentRoot: true + doc: | + Define a union type. + fields: + - name: documentRoot + type: boolean? + doc: | + If true, indicates that the type is a valid at the document root. At + least one type in a schema must be tagged with `documentRoot: true`. + jsonldPredicate: sld:documentRoot + + +- name: Documentation + type: record + docParent: "#Schema" + extends: [NamedType, DocType] + documentRoot: true + doc: | + A documentation section. This type exists to facilitate self-documenting + schemas but has no role in formal validation. + fields: + - name: type + doc: "Must be `documentation`" + type: + type: enum + name: Documentation_name + symbols: + - "sld:documentation" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/metaschema_base.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/metaschema_base.yml new file mode 100644 index 000000000..3bdf63903 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/metaschema_base.yml @@ -0,0 +1,273 @@ +$base: "https://w3id.org/cwl/salad#" + +$namespaces: + sld: "https://w3id.org/cwl/salad#" + dct: "http://purl.org/dc/terms/" + rdf: "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + rdfs: "http://www.w3.org/2000/01/rdf-schema#" + xsd: "http://www.w3.org/2001/XMLSchema#" + +$graph: + +- name: "Schema" + type: documentation + doc: | + # Schema + +- name: Documented + type: record + abstract: true + docParent: "#Schema" + fields: + - name: doc + type: + - string? + - string[]? + doc: "A documentation string for this object, or an array of strings which should be concatenated." + jsonldPredicate: "rdfs:comment" + + +- name: PrimitiveType + type: enum + symbols: + - "sld:null" + - "xsd:boolean" + - "xsd:int" + - "xsd:long" + - "xsd:float" + - "xsd:double" + - "xsd:string" + doc: + - | + Names of salad data types (based on Avro schema declarations). + + Refer to the [Avro schema declaration documentation](https://avro.apache.org/docs/current/spec.html#schemas) for + detailed information. + - "null: no value" + - "boolean: a binary value" + - "int: 32-bit signed integer" + - "long: 64-bit signed integer" + - "float: single precision (32-bit) IEEE 754 floating-point number" + - "double: double precision (64-bit) IEEE 754 floating-point number" + - "string: Unicode character sequence" + + +- name: Any + type: enum + symbols: ["#Any"] + docAfter: "#PrimitiveType" + doc: | + The **Any** type validates for any non-null value. + + +- name: RecordField + type: record + extends: Documented + doc: A field of a record. + fields: + - name: name + type: string + jsonldPredicate: "@id" + doc: | + The name of the field + + - name: type + type: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + - type: array + items: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + jsonldPredicate: + _id: sld:type + _type: "@vocab" + typeDSL: true + refScope: 2 + doc: | + The field type. If it is an array, it indicates + that the field type is a union type of its elements. + Its elements may be duplicated. + + +- name: RecordSchema + type: record + fields: + type: + doc: "Must be `record`" + type: + type: enum + name: Record_name + symbols: + - "sld:record" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 + fields: + type: RecordField[]? + jsonldPredicate: + _id: sld:fields + mapSubject: name + mapPredicate: type + doc: "Defines the fields of the record." + + +- name: EnumSchema + type: record + doc: | + Define an enumerated type. + fields: + type: + doc: "Must be `enum`" + type: + type: enum + name: Enum_name + symbols: + - "sld:enum" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 + name: + type: string? + jsonldPredicate: "@id" + symbols: + type: string[] + jsonldPredicate: + _id: "sld:symbols" + _type: "@id" + identity: true + doc: "Defines the set of valid symbols." + + +- name: ArraySchema + type: record + fields: + type: + doc: "Must be `array`" + type: + type: enum + name: Array_name + symbols: + - "sld:array" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 + items: + type: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + - type: array + items: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + jsonldPredicate: + _id: "sld:items" + _type: "@vocab" + refScope: 2 + doc: "Defines the type of the array elements." + + +- name: MapSchema + type: record + fields: + type: + doc: "Must be `map`" + type: + type: enum + name: Map_name + symbols: + - "sld:map" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 + values: + type: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + - type: array + items: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + jsonldPredicate: + _id: "sld:values" + _type: "@vocab" + refScope: 2 + doc: "Defines the type of the map elements." + + +- name: UnionSchema + type: record + fields: + type: + doc: "Must be `union`" + type: + type: enum + name: Union_name + symbols: + - "sld:union" + jsonldPredicate: + _id: "sld:type" + _type: "@vocab" + typeDSL: true + refScope: 2 + names: + type: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + - type: array + items: + - PrimitiveType + - RecordSchema + - EnumSchema + - ArraySchema + - MapSchema + - UnionSchema + - string + jsonldPredicate: + _id: "sld:names" + _type: "@vocab" + refScope: 2 + doc: "Defines the type of the union elements." diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/salad.md b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/salad.md new file mode 100644 index 000000000..26d4a6b8a --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/salad.md @@ -0,0 +1,354 @@ +# Semantic Annotations for Linked Avro Data (SALAD) + +Author: + +* Peter Amstutz , Curii Corporation + +Contributors: + +* The developers of Apache Avro +* The developers of JSON-LD +* Nebojša Tijanić , Seven Bridges Genomics +* Michael R. Crusoe, ELIXIR-DE +* Iacopo Colonnelli, University of Torino + +# Abstract + +Salad is a schema language for describing structured linked data documents +in JSON or YAML documents. A Salad schema provides rules for +preprocessing, structural validation, and link checking for documents +described by a Salad schema. Salad builds on JSON-LD and the Apache Avro +data serialization system, and extends Avro with features for rich data +modeling such as inheritance, template specialization, object identifiers, +and object references. Salad was developed to provide a bridge between the +record oriented data modeling supported by Apache Avro and the Semantic +Web. + +# Status of This Document + +This document is the product of the [Common Workflow Language working +group](https://groups.google.com/forum/#!forum/common-workflow-language). The +latest version of this document is available in the "schema_salad" repository at + +https://github.com/common-workflow-language/schema_salad + +The products of the CWL working group (including this document) are made available +under the terms of the Apache License, version 2.0. + + + +# Introduction + +The JSON data model is an extremely popular way to represent structured +data. It is attractive because of its relative simplicity and is a +natural fit with the standard types of many programming languages. +However, this simplicity means that basic JSON lacks expressive features +useful for working with complex data structures and document formats, such +as schemas, object references, and namespaces. + +JSON-LD is a W3C standard providing a way to describe how to interpret a +JSON document as Linked Data by means of a "context". JSON-LD provides a +powerful solution for representing object references and namespaces in JSON +based on standard web URIs, but is not itself a schema language. Without a +schema providing a well defined structure, it is difficult to process an +arbitrary JSON-LD document as idiomatic JSON because there are many ways to +express the same data that are logically equivalent but structurally +distinct. + +Several schema languages exist for describing and validating JSON data, +such as the Apache Avro data serialization system, however none understand +linked data. As a result, to fully take advantage of JSON-LD to build the +next generation of linked data applications, one must maintain separate +JSON schema, JSON-LD context, RDF schema, and human documentation, despite +significant overlap of content and obvious need for these documents to stay +synchronized. + +Schema Salad is designed to address this gap. It provides a schema +language and processing rules for describing structured JSON content +permitting URI resolution and strict document validation. The schema +language supports linked data through annotations that describe the linked +data interpretation of the content, enables generation of JSON-LD context +and RDF schema, and production of RDF triples by applying the JSON-LD +context. The schema language also provides for robust support of inline +documentation. + +## Introduction to v1.1 + +This is the third version of the Schema Salad specification. It is +developed concurrently with v1.1 of the Common Workflow Language for use in +specifying the Common Workflow Language, however Schema Salad is intended to be +useful to a broader audience. Compared to the v1.0 schema salad +specification, the following changes have been made: + +* Support for `default` values on record fields to specify default values +* Add subscoped fields (fields which introduce a new inner scope for identifiers) +* Add the *inVocab* flag (default true) to indicate if a type is added to the vocabulary of well known terms or must be prefixed +* Add *secondaryFilesDSL* micro DSL (domain specific language) to convert text strings to a secondaryFiles record type used in CWL +* The `$mixin` feature has been removed from the specification, as it + is poorly documented, not included in conformance testing, + and not widely supported. + +## Introduction to v1.2 + +This is the fourth version of the Schema Salad specification. It was created to +ease the development of extensions to CWL v1.2. The only change is that +inherited records can narrow the types of fields if those fields are re-specified +with a matching jsonldPredicate. + +### Changelog for v1.2.1 + +There are no new features nor behavior changes in Schema Salad v1.2.1 +as compared to Schema-Salad v1.2. Schema Salad v1.2.1 only fixes typos and adds +clarifications. + +* The `salad` directory's contents have been trimmed to the bare necessities. + The `salad/README.rst` has been refreshed from the [upstream repository](https://github.com/common-workflow-language/schema_salad/). +* The [existing behaviour of `$import`](#Import) has been clarified. + If the `$import` node is in an array and the import operation yields an + array, it is flattened to the parent array. Otherwise the `$import` + node is replaced in the document structure by the object or array yielded + from the import operation. An [additional example](#import_example2) + has been added to illustrate this better. +* A pair of missing brackets was added to the [Type DSL Example](#Type_DSL_example)'s + example input. +* Missing newlines have been added to the [identifier map example](#Identifier_map_example)'s + example source and example result. +* [Inherited fields in Salad types](#SaladRecordSchema) may be re-specified + to narrow their type and/or to override the `doc` field. +* Clarify that fields with `jsonldPredicate: { _type: "@id" }` indicate that the + field is a [link fields](#SaladRecordSchema) and that if the `jsonldPredicate` + also has the field `identity` with the value `true`, then field is + resolved with [identifier resolution](#Identifier_resolution). + Otherwise the field is resolved with [link resolution](#Link_resolution). + +## Introduction to v1.3 + +This is the fifth version of the Schema Salad specification. It was created to +enhance code generation by representing CWL data types as specific Python objects +(instead of relying on the generic `Any` type). The following changes have been made: + +* Support for the Avro `map` schema +* Add named versions of the `map` and `union` Avro types +* Support for nested named `union` type definitions + +## References to Other Specifications + +**Javascript Object Notation (JSON)**: http://json.org + +**JSON Linked Data (JSON-LD)**: http://json-ld.org + +**YAML**: https://yaml.org/spec/1.2/spec.html + +**Avro**: https://avro.apache.org/docs/current/spec.html + +**Uniform Resource Identifier (URI) Generic Syntax**: https://tools.ietf.org/html/rfc3986) + +**Resource Description Framework (RDF)**: http://www.w3.org/RDF/ + +**UTF-8**: https://www.ietf.org/rfc/rfc2279.txt) + +## Scope + +This document describes the syntax, data model, algorithms, and schema +language for working with Salad documents. It is not intended to document +a specific implementation of Salad, however it may serve as a reference for +the behavior of conforming implementations. + +## Terminology + +The terminology used to describe Salad documents is defined in the Concepts +section of the specification. The terms defined in the following list are +used in building those definitions and in describing the actions of a +Salad implementation: + +**may**: Conforming Salad documents and Salad implementations are permitted but +not required to be interpreted as described. + +**must**: Conforming Salad documents and Salad implementations are required +to be interpreted as described; otherwise they are in error. + +**error**: A violation of the rules of this specification; results are +undefined. Conforming implementations may detect and report an error and may +recover from it. + +**fatal error**: A violation of the rules of this specification; results +are undefined. Conforming implementations must not continue to process the +document and may report an error. + +**at user option**: Conforming software may or must (depending on the modal verb in +the sentence) behave as described; if it does, it must provide users a means to +enable or disable the behavior described. + +# Document model + +## Data concepts + +An **object** is a data structure equivalent to the "object" type in JSON, +consisting of a unordered set of name/value pairs (referred to here as +**fields**) and where the name is a string and the value is a string, number, +boolean, array, or object. + +A **document** is a file containing a serialized object, or an array of +objects. + +A **document type** is a class of files that share a common structure and +semantics. + +A **document schema** is a formal description of the grammar of a document type. + +A **base URI** is a context-dependent URI used to resolve relative references. + +An **identifier** is a URI that designates a single document or single +object within a document. + +A **vocabulary** is the set of symbolic field names and enumerated symbols defined +by a document schema, where each term maps to absolute URI. + +## Syntax + +Conforming Salad v1.1 documents are serialized and loaded using a +subset of YAML 1.2 syntax and UTF-8 text encoding. Salad documents +are written using the [JSON-compatible subset of YAML described in +section 10.2](https://yaml.org/spec/1.2/spec.html#id2803231). The +following features of YAML must not be used in conforming Salad +documents: + +* Use of explicit node tags with leading `!` or `!!` +* Use of anchors with leading `&` and aliases with leading `*` +* %YAML directives +* %TAG directives + +It is a fatal error if the document is not valid YAML. + +A Salad document must consist only of either a single root object or an +array of objects. + +## Document context + +### Implied context + +The implicit context consists of the vocabulary defined by the schema and +the base URI. By default, the base URI must be the URI that was used to +load the document. It may be overridden by an explicit context. + +### Explicit context + +If a document consists of a root object, this object may contain the +fields `$base`, `$namespaces`, `$schemas`, and `$graph`: + + * `$base`: Must be a string. Set the base URI for the document used to + resolve relative references. + + * `$namespaces`: Must be an object with strings as values. The keys of + the object are namespace prefixes used in the document; the values of + the object are the prefix expansions. + + * `$schemas`: Must be an array of strings. This field may list URI + references to documents in RDF-XML format which will be queried for RDF + schema data. The subjects and predicates described by the RDF schema + may provide additional semantic context for the document, and may be + used for validation of prefixed extension fields found in the document. + +Other directives beginning with `$` must be ignored. + +## Document graph + +If a document consists of a single root object, this object may contain the +field `$graph`. This field must be an array of objects. If present, this +field holds the primary content of the document. A document that consists +of array of objects at the root is an implicit graph. + +## Document metadata + +If a document consists of a single root object, metadata about the +document, such as authorship, may be declared in the root object. + +## Document schema + +Document preprocessing, link validation and schema validation require a +document schema. A schema may consist of: + + * At least one record definition object which defines valid fields that + make up a record type. Record field definitions include the valid types + that may be assigned to each field and annotations to indicate fields + that represent identifiers and links, described below in "Semantic + Annotations". + + * Any number of enumerated type objects which define a set of finite set of symbols that are + valid value of the type. + + * Any number of documentation objects which allow in-line documentation of the schema. + +The schema for defining a salad schema (the metaschema) is described in +detail in the [Schema](#Schema) section. + +## Record field annotations + +In a document schema, record field definitions may include the field +`jsonldPredicate`, which may be either a string or object. Implementations +must use the following document preprocessing of fields by the following +rules: + + * If the value of `jsonldPredicate` is `@id`, the field is an identifier + field. + + * If the value of `jsonldPredicate` is an object, and that + object contains the field `_type` with the value `@id`, the + field is a link field. If the field `jsonldPredicate` also + has the field `identity` with the value `true`, the field is + resolved with [identifier resolution](#Identifier_resolution). + Otherwise it is resolved with [link resolution](#Link_resolution). + + * If the value of `jsonldPredicate` is an object which contains the + field `_type` with the value `@vocab`, the field value is subject to + [vocabulary resolution](#Vocabulary_resolution). + +## Document traversal + +To perform document preprocessing, link validation and schema +validation, the document must be traversed starting from the fields or +array items of the root object or array and recursively visiting each child +item which contains an object or arrays. + +## Short names + +The "short name" of a fully qualified identifier is the portion of +the identifier following the final slash `/` of either the fragment +identifier following `#` or the path portion, if there is no fragment. +Some examples: + +* the short name of `http://example.com/foo` is `foo` +* the short name of `http://example.com/#bar` is `bar` +* the short name of `http://example.com/foo/bar` is `bar` +* the short name of `http://example.com/foo#bar` is `bar` +* the short name of `http://example.com/#foo/bar` is `bar` +* the short name of `http://example.com/foo#bar/baz` is `baz` + +## Inheritance and specialization + +A record definition may inherit from one or more record definitions +with the `extends` field. This copies the fields defined in the +parent record(s) as the base for the new record. A record definition +may `specialize` type declarations of the fields inherited from the +base record. For each field inherited from the base record, any +instance of the type in `specializeFrom` is replaced with the type in +`specializeTo`. The type in `specializeTo` should extend from the +type in `specializeFrom`. + +A record definition may be `abstract`. This means the record +definition is not used for validation on its own, but may be extended +by other definitions. If an abstract type appears in a field +definition, it is logically replaced with a union of all concrete +subtypes of the abstract type. In other words, the field value does +not validate as the abstract type, but must validate as some concrete +type that inherits from the abstract type. + +# Document preprocessing + +After processing the explicit context (if any), document preprocessing +begins. Starting from the document root, object fields values or array +items which contain objects or arrays are recursively traversed +depth-first. For each visited object, field names, identifier fields, link +fields, vocabulary fields, and `$import` and `$include` directives must be +processed as described in this section. The order of traversal of child +nodes within a parent node is undefined. diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res.yml new file mode 100644 index 000000000..f61949508 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res.yml @@ -0,0 +1,34 @@ +- | + ## Domain Specific Language for secondary files + + Fields may be tagged `secondaryFilesDSL: true` in `jsonldPredicate`. If so, the field is expanded using the + following micro-DSL for secondary files: + + * If the value is a string, it is transformed to an object with two fields `pattern` and `required` + * By default, the value of `required` is `null` (this indicates default behavior, which may be based on the context) + * If the value ends with a question mark `?` the question mark is + stripped off and the value of the field `required` is set to `False` + * The remaining value is assigned to the field `pattern` + + ### Type DSL example + + Given the following schema: + + ``` +- $include: sfdsl_res_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: sfdsl_res_src.yml +- | + ``` + + This becomes: + + ``` +- $include: sfdsl_res_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_proc.yml new file mode 100644 index 000000000..2d27a687c --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_proc.yml @@ -0,0 +1,21 @@ +[ + { + "secondaryFiles": { + "pattern": ".bai", + "required": null + }, + { + "secondaryFiles": { + "pattern": ".bai", + "required": false + }, + { + "secondaryFiles": { + "pattern": ".bai?" + }, + { + "secondaryFiles": { + "pattern": ".bai?", + "required": true + }, +] diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_schema.yml new file mode 100644 index 000000000..549128a81 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_schema.yml @@ -0,0 +1,16 @@ +{ + "$graph": [ + { + "name": "SecondaryFilesDSLExample", + "type": "record", + "documentRoot": true, + "fields": [{ + "name": "secondaryFiles", + "type": "string", + "jsonldPredicate": { + _type: "@vocab", + "secondaryFilesDSL": true + } + }] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_src.yml new file mode 100644 index 000000000..c12869cba --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/sfdsl_res_src.yml @@ -0,0 +1,12 @@ +[{ + "secondaryFiles": ".bai" +}, { + "secondaryFiles": ".bai?" +}, { + "secondaryFiles": { + "pattern": ".bai?" +}}, { + "secondaryFiles": { + "pattern": ".bai?", + "required": true +}}] diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res.yml new file mode 100644 index 000000000..8d347b840 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res.yml @@ -0,0 +1,38 @@ +- | + ## Domain Specific Language for types + + Fields may be tagged `typeDSL: true` in `jsonldPredicate`. If so, the field is expanded using the + following micro-DSL for schema salad types: + + * If the type ends with a question mark `?`, the question mark is stripped off and the type is expanded to a union with `null` + * If the type ends with square brackets `[]` it is expanded to an array with items of the preceding type symbol + * The type may end with both square brackets with one question mark (`[]?`) to indicate it is an optional array. + * Identifier resolution is applied after type DSL expansion. + + Starting with Schema Salad version 1.3, fields tagged with `typeDSL: true` in `jsonldPredicate` have the following additional behavior: + + * Square brackes `[]` can be repeated to indicate 2, 3, or more dimensional array types. + * These multi-dimensional arrays, like 1-dimensional arrays, can be combined with `?` (for example, `[][]?`) to indicate that it is an optional multi-dimensional array. + + ### Type DSL example + + Given the following schema: + + ``` +- $include: typedsl_res_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: typedsl_res_src.yml +- | + ``` + + This becomes: + + ``` +- $include: typedsl_res_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_proc.yml new file mode 100644 index 000000000..8097a6ac5 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_proc.yml @@ -0,0 +1,26 @@ +[ + { + "extype": "string" + }, + { + "extype": [ + "null", + "string" + ] + }, + { + "extype": { + "type": "array", + "items": "string" + } + }, + { + "extype": [ + "null", + { + "type": "array", + "items": "string" + } + ] + } +] diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_schema.yml new file mode 100644 index 000000000..52459a657 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_schema.yml @@ -0,0 +1,17 @@ +{ + "$graph": [ + {"$import": "metaschema_base.yml"}, + { + "name": "TypeDSLExample", + "type": "record", + "documentRoot": true, + "fields": [{ + "name": "extype", + "type": "string", + "jsonldPredicate": { + _type: "@vocab", + "typeDSL": true + } + }] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_src.yml new file mode 100644 index 000000000..6ecbd50d1 --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/typedsl_res_src.yml @@ -0,0 +1,9 @@ +[{ + "extype": "string" +}, { + "extype": "string?" +}, { + "extype": "string[]" +}, { + "extype": "string[]?" +}] diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res.yml new file mode 100644 index 000000000..b9d5f53bd --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res.yml @@ -0,0 +1,36 @@ +- | + ## Vocabulary resolution + + The schema may designate one or more vocabulary fields which use + terms defined in the vocabulary. The vocabulary are the short + names of all the identifiers in the schema. Processing must + resolve vocabulary fields to either vocabulary terms or absolute + URIs by first applying the link resolution rules defined above, + then applying the following additional rule: + + * If a reference URI is a vocabulary field, and there is a vocabulary + term which maps to the resolved URI, the reference must be replaced with + the vocabulary term. + + ### Vocabulary resolution example + + Given the following schema: + + ``` +- $include: vocab_res_schema.yml +- | + ``` + + Process the following example: + + ``` +- $include: vocab_res_src.yml +- | + ``` + + This becomes: + + ``` +- $include: vocab_res_proc.yml +- | + ``` diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_proc.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_proc.yml new file mode 100644 index 000000000..d13ab151f --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_proc.yml @@ -0,0 +1,15 @@ + { + "form": { + "things": [ + { + "voc": "red", + }, + { + "voc": "red", + }, + { + "voc": "http://example.com/acid#blue", + } + ] + } + } diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_schema.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_schema.yml new file mode 100644 index 000000000..92b271e8a --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_schema.yml @@ -0,0 +1,21 @@ +{ + "$namespaces": { + "acid": "http://example.com/acid#" + }, + "$graph": [{ + "name": "Colors", + "type": "enum", + "symbols": ["acid:red"] + }, + { + "name": "ExampleType", + "type": "record", + "fields": [{ + "name": "voc", + "type": "string", + "jsonldPredicate": { + "_type": "@vocab" + } + }] + }] +} diff --git a/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_src.yml b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_src.yml new file mode 100644 index 000000000..82954f14a --- /dev/null +++ b/cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/vocab_res_src.yml @@ -0,0 +1,15 @@ + { + "form": { + "things": [ + { + "voc": "red", + }, + { + "voc": "http://example.com/acid#red", + }, + { + "voc": "http://example.com/acid#blue", + } + ] + } + } diff --git a/cwltool/update.py b/cwltool/update.py index a3e8569c3..7b801f5dc 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -11,15 +11,63 @@ cast, ) -from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader from schema_salad.sourceline import SourceLine +from ruamel.yaml.comments import CommentedMap, CommentedSeq + from .loghandler import _logger from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field +def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> Tuple[CommentedMap, str]: + """Public updater for v1.2 to v1.3.0-dev1.""" + doc = copy.deepcopy(doc) + + def rewrite_loop_requirements(t: CWLObjectType) -> None: + for s in cast(MutableSequence[CWLObjectType], t["steps"]): + if "requirements" in s: + for i, r in enumerate( + list(cast(MutableSequence[CWLObjectType], s["requirements"])) + ): + cls = cast(str, r["class"]) + if cls == "http://commonwl.org/cwltool#Loop": + if "when" in s: + raise SourceLine(s, "when", ValidationException).makeError( + "The `cwltool:Loop` clause is not compatible with the `when` directive." + ) + if "loopWhen" not in r: + raise SourceLine( + r, raise_type=ValidationException + ).makeError( # pragma: no cover + "The `loopWhen` clause is mandatory within the `cwltool:Loop` requirement." + ) + s["when"] = r["loopWhen"] + if "loop" in r: + for el in cast(MutableSequence[CWLObjectType], r["loop"]): + source = el.pop("loopSource", None) + if source is not None: + el["outputSource"] = source + s["loop"] = r["loop"] + if "outputMethod" in r: + s["outputMethod"] = r["outputMethod"] + cast( + MutableSequence[CWLObjectType], + s["requirements"], + ).pop(index=i) + if "hints" in s: + for r in cast(MutableSequence[CWLObjectType], s["hints"]): + cls = cast(str, r["class"]) + if cls == "http://commonwl.org/cwltool#Loop": + raise SourceLine(s["hints"], r, ValidationException).makeError( + "http://commonwl.org/cwltool#Loop is valid only under requirements." + ) + + visit_class(doc, "Workflow", rewrite_loop_requirements) + return doc, "v1.3.0-dev1" + + def v1_1to1_2( doc: CommentedMap, loader: Loader, baseuri: str ) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument @@ -205,12 +253,13 @@ def v1_2_0dev5to1_2( "v1.2.0-dev4", "v1.2.0-dev5", "v1.2", + "v1.3.0-dev1", ] UPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, - "v1.2": None, + "v1.2": v1_2to1_3dev1, } DEVUPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { @@ -220,13 +269,14 @@ def v1_2_0dev5to1_2( "v1.2.0-dev3": v1_2_0dev3todev4, "v1.2.0-dev4": v1_2_0dev4todev5, "v1.2.0-dev5": v1_2_0dev5to1_2, + "v1.3.0-dev1": None, } ALLUPDATES = UPDATES.copy() ALLUPDATES.update(DEVUPDATES) -INTERNAL_VERSION = "v1.2" +INTERNAL_VERSION = "v1.3.0-dev1" ORIGINAL_CWLVERSION = "http://commonwl.org/cwltool#original_cwlVersion" diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 8546ca72e..727331dd4 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -6,13 +6,11 @@ from typing import ( Callable, Dict, - Iterable, List, Mapping, MutableMapping, MutableSequence, Optional, - Union, cast, ) from uuid import UUID @@ -33,7 +31,6 @@ from .process import Process, get_overrides, shortname from .utils import ( CWLObjectType, - CWLOutputType, JobsGeneratorType, OutputCallbackType, StepType, @@ -219,7 +216,7 @@ def __init__( if parent_req["class"] == step_req["class"]: found_in_step = True break - if not found_in_step and parent_req.get("class") != "http://commonwl.org/cwltool#Loop": + if not found_in_step: loadingContext.requirements.append(parent_req) loadingContext.requirements.extend( cast( @@ -405,16 +402,6 @@ def __init__( else: self.parent_wf = self.prov_obj - def checkRequirements( - self, - rec: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType, None], - supported_process_requirements: Iterable[str], - ) -> None: - """Check the presence of unsupported requirements.""" - supported_process_requirements = list(supported_process_requirements) - supported_process_requirements.append("http://commonwl.org/cwltool#Loop") - super().checkRequirements(rec, supported_process_requirements) - def receive_output( self, output_callback: OutputCallbackType, diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index c85e29516..e6c1f8d8a 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -734,7 +734,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] step.name, json_dumps(inputobj, indent=4), ) - if step.step.get_requirement("http://commonwl.org/cwltool#Loop")[0]: + if step.tool.get("loop"): jobs = WorkflowJobLoopStep( step=step, container_engine=container_engine ).job(inputobj, callback, runtimeContext) @@ -742,7 +742,13 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] jobs = step.job(inputobj, callback, runtimeContext) else: _logger.info("[%s] will be skipped", step.name) - callback({k["id"]: None for k in outputparms}, "skipped") + if ( + step.tool.get("loop") is not None + and step.tool.get("outputMethod", "last") == "all" + ): + callback({k["id"]: [] for k in outputparms}, "skipped") + else: + callback({k["id"]: None for k in outputparms}, "skipped") step.completed = True jobs = (_ for _ in ()) @@ -850,7 +856,7 @@ def job( class WorkflowJobLoopStep: - """Generated for each step in Workflow.steps() containing a Loop requirement.""" + """Generated for each step in Workflow.steps() containing a `loop` directive.""" def __init__(self, step: WorkflowJobStep, container_engine: str): """Initialize this WorkflowJobLoopStep.""" @@ -864,11 +870,11 @@ def __init__(self, step: WorkflowJobStep, container_engine: str): Union[MutableSequence[Optional[CWLOutputType]], Optional[CWLOutputType]], ] = {} - def _set_empty_output(self, loop_req: CWLObjectType) -> None: + def _set_empty_output(self, outputMethod: str) -> None: for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) - if loop_req.get("outputMethod") == "all": + if outputMethod == "all": self.output_buffer[iid] = cast(MutableSequence[Optional[CWLOutputType]], []) else: self.output_buffer[iid] = None @@ -879,12 +885,9 @@ def job( output_callback: OutputCallbackType, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: - """Generate a WorkflowJobStep job until the `loopWhen` condition evaluates to False.""" + """Generate a WorkflowJobStep job until the `when` condition evaluates to False.""" self.joborder = joborder - loop_req = cast( - CWLObjectType, - self.step.step.get_requirement("http://commonwl.org/cwltool#Loop")[0], - ) + outputMethod = self.step.tool.get("outputMethod", "last") callback = functools.partial( self.loop_callback, @@ -895,7 +898,7 @@ def job( while True: evalinputs = {shortname(k): v for k, v in self.joborder.items()} whenval = expression.do_eval( - loop_req["loopWhen"], + self.step.tool["when"], evalinputs, self.step.step.requirements, None, @@ -916,9 +919,9 @@ def job( return elif whenval is False: _logger.debug( - "[%s] loop condition %s evaluated to %s at iteration %i", + "[%s] condition %s evaluated to %s at iteration %i", self.step.name, - loop_req["loopWhen"], + self.step.tool["when"], whenval, self.iteration, ) @@ -927,22 +930,17 @@ def job( self.step.name, json_dumps(evalinputs, indent=2), ) - if self.iteration == 0: - self.processStatus = "skipped" - self._set_empty_output(loop_req) output_callback(self.output_buffer, self.processStatus) return else: - raise WorkflowException( - "Loop condition 'loopWhen' must evaluate to 'true' or 'false'" - ) + raise WorkflowException("Conditional 'when' must evaluate to 'true' or 'false'") except WorkflowException: raise except Exception: _logger.exception("Unhandled exception") self.processStatus = "permanentFail" if self.iteration == 0: - self._set_empty_output(loop_req) + self._set_empty_output(outputMethod) # pragma: no cover output_callback(self.output_buffer, self.processStatus) def loop_callback( @@ -954,17 +952,15 @@ def loop_callback( """Update the joborder object with output values from the last iteration.""" self.iteration += 1 try: - loop_req = cast( - CWLObjectType, - self.step.step.get_requirement("http://commonwl.org/cwltool#Loop")[0], - ) + loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) + outputMethod = self.step.tool.get("outputMethod", "last") state: Dict[str, Optional[WorkflowStateItem]] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) if iid in jobout: state[iid] = WorkflowStateItem(i, jobout[iid], processStatus) - if loop_req.get("outputMethod") == "all": + if outputMethod == "all": if iid not in self.output_buffer: self.output_buffer[iid] = cast( MutableSequence[Optional[CWLOutputType]], [] @@ -994,7 +990,7 @@ def loop_callback( if self.processStatus != "permanentFail": self.processStatus = processStatus - if processStatus not in ("success", "skipped"): + if processStatus != "success": _logger.warning( "[%s] Iteration %i completed %s", self.step.name, @@ -1019,26 +1015,17 @@ def loop_callback( CWLObjectType, object_from_state( state, - [ - {**source, **{"type": "Any"}} - for source in cast( - MutableSequence[CWLObjectType], loop_req.get("loop", []) - ) - ], + [{**source, **{"type": "Any"}} for source in loop], False, supportsMultipleInput, - "loopSource", + "outputSource", ), ), } fs_access = getdefault(runtimeContext.make_fs_access, StdFsAccess)("") - valueFrom = { - i["id"]: i["valueFrom"] - for i in cast(MutableSequence[CWLObjectType], loop_req.get("loop", [])) - if "valueFrom" in i - } + valueFrom = {i["id"]: i["valueFrom"] for i in loop if "valueFrom" in i} if len(valueFrom) > 0 and not bool( self.step.step.get_requirement("StepInputExpressionRequirement")[0] ): diff --git a/mypy-requirements.txt b/mypy-requirements.txt index c2f3de419..31c4c0211 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,9 +1,9 @@ -mypy==1.10.0 # also update pyproject.toml +mypy==1.10.1 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 types-requests types-setuptools types-psutil types-mock -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.1 -galaxy-util<24.1 +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 +galaxy-util<24.2 diff --git a/pyproject.toml b/pyproject.toml index e845a4e5f..1753a6175 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,12 +2,12 @@ requires = [ "setuptools>=45", "setuptools_scm[toml]>=8.0.4,<9", - "mypy==1.10.0", # also update mypy-requirements.txt + "mypy==1.10.1", # also update mypy-requirements.txt "types-requests", "types-psutil", "importlib_resources>=1.4;python_version<'3.9'", "ruamel.yaml>=0.16.0,<0.18", - "schema-salad>=8.4.20230426093816,<9", + "schema-salad>=8.7,<9", "cwl-utils>=0.32", "toml", "argcomplete>=1.12.0", diff --git a/requirements.txt b/requirements.txt index 036c4eed6..df82a6f43 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,13 @@ requests>=2.6.1 ruamel.yaml>=0.16.0,<0.19 rdflib>=4.2.2,<7.1 shellescape>=3.4.1,<3.9 -schema-salad>=8.4.20230426093816,<9 +schema-salad>=8.7,<9 prov==1.5.1 mypy-extensions psutil>=5.6.6 importlib_resources>=1.4;python_version<'3.9' coloredlogs -pydot>=1.4.1 +pydot>=1.4.1,<3 argcomplete>=1.12.0 pyparsing!=3.0.2 # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 cwl-utils>=0.32 diff --git a/setup.py b/setup.py index 896dd7a61..cd74658e8 100644 --- a/setup.py +++ b/setup.py @@ -127,13 +127,13 @@ "ruamel.yaml >= 0.16, < 0.19", "rdflib >= 4.2.2, < 7.1.0", "shellescape >= 3.4.1, < 3.9", - "schema-salad >= 8.4.20230426093816, < 9", + "schema-salad >= 8.7, < 9", "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", - "pydot >= 1.4.1", + "pydot >= 1.4.1, <3", "argcomplete", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", @@ -141,8 +141,8 @@ ], extras_require={ "deps": [ - "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.1", - "galaxy-util <24.1", + "galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2", + "galaxy-util <24.2", ], }, python_requires=">=3.8, <4", diff --git a/test-requirements.txt b/test-requirements.txt index cffff68a3..ce709ef96 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,6 +8,6 @@ mock>=2.0.0 pytest-mock>=1.10.0 pytest-cov arcp>=0.2.0 --rrequirements.txt -galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.1 -galaxy-util<24.1 +-r requirements.txt +galaxy-tool-util>=22.1.2,!=23.0.1,!=23.0.2,!=23.0.3,!=23.0.4,!=23.0.5,<24.2 +galaxy-util<24.2 diff --git a/tests/loop-ext/all-output-loop-no-iteration.cwl b/tests/loop-ext/all-output-loop-no-iteration.cwl new file mode 100644 index 000000000..be5e6468e --- /dev/null +++ b/tests/loop-ext/all-output-loop-no-iteration.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: all diff --git a/tests/loop-ext/all-output-loop.cwl b/tests/loop-ext/all-output-loop.cwl new file mode 100644 index 000000000..096be5192 --- /dev/null +++ b/tests/loop-ext/all-output-loop.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: all diff --git a/tests/loop-ext/default-value-loop.cwl b/tests/loop-ext/default-value-loop.cwl new file mode 100644 index 000000000..453a7613e --- /dev/null +++ b/tests/loop-ext/default-value-loop.cwl @@ -0,0 +1,51 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: loop/o1 + pickValue: all_non_null +steps: + loop: + run: + class: Workflow + inputs: + i1: int + outputs: + o1: + type: int? + outputSource: big_values/o1 + steps: + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [ o1 ] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 20) + loop: + i1: + loopSource: o1 + default: 5 + outputMethod: all \ No newline at end of file diff --git a/tests/loop/invalid-loop-command-line-tool.cwl b/tests/loop-ext/invalid-loop-command-line-tool.cwl similarity index 100% rename from tests/loop/invalid-loop-command-line-tool.cwl rename to tests/loop-ext/invalid-loop-command-line-tool.cwl diff --git a/tests/loop/invalid-loop-expression-tool.cwl b/tests/loop-ext/invalid-loop-expression-tool.cwl similarity index 100% rename from tests/loop/invalid-loop-expression-tool.cwl rename to tests/loop-ext/invalid-loop-expression-tool.cwl diff --git a/tests/loop/invalid-loop-hint.cwl b/tests/loop-ext/invalid-loop-hint.cwl similarity index 100% rename from tests/loop/invalid-loop-hint.cwl rename to tests/loop-ext/invalid-loop-hint.cwl diff --git a/tests/loop-ext/invalid-loop-scatter.cwl b/tests/loop-ext/invalid-loop-scatter.cwl new file mode 100644 index 000000000..fa6fa248c --- /dev/null +++ b/tests/loop-ext/invalid-loop-scatter.cwl @@ -0,0 +1,38 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int[] + i2: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + scatter: i1 + out: [o1] diff --git a/tests/loop-ext/invalid-loop-when-exception.cwl b/tests/loop-ext/invalid-loop-when-exception.cwl new file mode 100644 index 000000000..13bce53df --- /dev/null +++ b/tests/loop-ext/invalid-loop-when-exception.cwl @@ -0,0 +1,40 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + requirements: + cwltool:Loop: + loopWhen: | + ${ + throw new Error("failed"); + } + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/invalid-loop-when.cwl b/tests/loop-ext/invalid-loop-when.cwl similarity index 100% rename from tests/loop/invalid-loop-when.cwl rename to tests/loop-ext/invalid-loop-when.cwl diff --git a/tests/loop/invalid-loop-workflow.cwl b/tests/loop-ext/invalid-loop-workflow.cwl similarity index 100% rename from tests/loop/invalid-loop-workflow.cwl rename to tests/loop-ext/invalid-loop-workflow.cwl diff --git a/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl b/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl new file mode 100644 index 000000000..c56ada500 --- /dev/null +++ b/tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl @@ -0,0 +1,68 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened + pickValue: all_non_null +steps: + loop: + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 20) + loop: + i1: + loopSource: [osmall, obig] + pickValue: the_only_non_null + outputMethod: all \ No newline at end of file diff --git a/tests/loop-ext/invalid-no-loopWhen.cwl b/tests/loop-ext/invalid-no-loopWhen.cwl new file mode 100644 index 000000000..c1f52f6f4 --- /dev/null +++ b/tests/loop-ext/invalid-no-loopWhen.cwl @@ -0,0 +1,36 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + requirements: + cwltool:Loop: + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen.cwl new file mode 100644 index 000000000..99794d1e8 --- /dev/null +++ b/tests/loop-ext/invalid-non-boolean-loopWhen.cwl @@ -0,0 +1,37 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1) + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl b/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl new file mode 100644 index 000000000..4f06aa5b6 --- /dev/null +++ b/tests/loop-ext/invalid-non-boolean-loopWhen2.cwl @@ -0,0 +1,37 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + requirements: + cwltool:Loop: + loopWhen: '$(inputs.i1 == 1 ? true : "I am a string")' + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl b/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl new file mode 100644 index 000000000..d2d7668ad --- /dev/null +++ b/tests/loop-ext/invalid-value-from-loop-no-requirement.cwl @@ -0,0 +1,36 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last diff --git a/tests/loop-ext/loop-inside-loop-all.cwl b/tests/loop-ext/loop-inside-loop-all.cwl new file mode 100644 index 000000000..f62ea177c --- /dev/null +++ b/tests/loop-ext/loop-inside-loop-all.cwl @@ -0,0 +1,64 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: + type: array + items: + type: array + items: int + outputSource: loop1/o1 +steps: + loop1: + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int[] + outputSource: loop2/o1 + steps: + loop2: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: all + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all diff --git a/tests/loop-ext/loop-inside-loop.cwl b/tests/loop-ext/loop-inside-loop.cwl new file mode 100644 index 000000000..223fe542b --- /dev/null +++ b/tests/loop-ext/loop-inside-loop.cwl @@ -0,0 +1,60 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int[] + outputSource: loop1/o1 +steps: + loop1: + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int + outputSource: loop2/o1 + steps: + loop2: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all diff --git a/tests/loop-ext/loop-inside-scatter-job.yml b/tests/loop-ext/loop-inside-scatter-job.yml new file mode 100644 index 000000000..0d95d06fd --- /dev/null +++ b/tests/loop-ext/loop-inside-scatter-job.yml @@ -0,0 +1,2 @@ +i1: [1, 2, 3, 4, 5] +i2: 1 \ No newline at end of file diff --git a/tests/loop-ext/loop-inside-scatter.cwl b/tests/loop-ext/loop-inside-scatter.cwl new file mode 100644 index 000000000..358309e23 --- /dev/null +++ b/tests/loop-ext/loop-inside-scatter.cwl @@ -0,0 +1,53 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int[] + i2: int +outputs: + o1: + type: int[] + outputSource: scatter/o1 +steps: + scatter: + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int + outputSource: subworkflow/o1 + steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + scatter: i1 + out: [o1] diff --git a/tests/loop-ext/multi-source-loop.cwl b/tests/loop-ext/multi-source-loop.cwl new file mode 100644 index 000000000..7f72db224 --- /dev/null +++ b/tests/loop-ext/multi-source-loop.cwl @@ -0,0 +1,69 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + MultipleInputFeatureRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened + pickValue: all_non_null +steps: + loop: + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 20) + loop: + i1: + loopSource: [osmall, obig] + pickValue: the_only_non_null + outputMethod: all \ No newline at end of file diff --git a/tests/loop-ext/opt-var-loop.cwl b/tests/loop-ext/opt-var-loop.cwl new file mode 100644 index 000000000..3d9e51afe --- /dev/null +++ b/tests/loop-ext/opt-var-loop.cwl @@ -0,0 +1,34 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int? + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop-ext/scatter-inside-loop.cwl b/tests/loop-ext/scatter-inside-loop.cwl new file mode 100644 index 000000000..7a4c394be --- /dev/null +++ b/tests/loop-ext/scatter-inside-loop.cwl @@ -0,0 +1,53 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int[] + i2: int +outputs: + o1: + type: int[] + outputSource: scatter/o1 +steps: + scatter: + run: + class: Workflow + inputs: + i1: int[] + i2: int + outputs: + o1: + type: int[] + outputSource: subworkflow/o1 + steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + scatter: i1 + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1[0] < 10) + loop: + i1: o1 + outputMethod: last \ No newline at end of file diff --git a/tests/loop-ext/single-var-loop-job.yml b/tests/loop-ext/single-var-loop-job.yml new file mode 100644 index 000000000..d491f14ca --- /dev/null +++ b/tests/loop-ext/single-var-loop-job.yml @@ -0,0 +1 @@ +i1: 1 \ No newline at end of file diff --git a/tests/loop-ext/single-var-loop-no-iteration.cwl b/tests/loop-ext/single-var-loop-no-iteration.cwl new file mode 100644 index 000000000..451083a4b --- /dev/null +++ b/tests/loop-ext/single-var-loop-no-iteration.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop-ext/single-var-loop.cwl b/tests/loop-ext/single-var-loop.cwl new file mode 100644 index 000000000..0feac136d --- /dev/null +++ b/tests/loop-ext/single-var-loop.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop-ext/two-vars-loop-2.cwl b/tests/loop-ext/two-vars-loop-2.cwl new file mode 100644 index 000000000..bcc589215 --- /dev/null +++ b/tests/loop-ext/two-vars-loop-2.cwl @@ -0,0 +1,35 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop-ext/two-vars-loop-job.yml b/tests/loop-ext/two-vars-loop-job.yml new file mode 100644 index 000000000..012ed44ac --- /dev/null +++ b/tests/loop-ext/two-vars-loop-job.yml @@ -0,0 +1,2 @@ +i1: 1 +i2: 1 \ No newline at end of file diff --git a/tests/loop-ext/two-vars-loop.cwl b/tests/loop-ext/two-vars-loop.cwl new file mode 100644 index 000000000..2a16ae7db --- /dev/null +++ b/tests/loop-ext/two-vars-loop.cwl @@ -0,0 +1,37 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + o2: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2, 'o2': inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1, o2] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: o1 + i2: o2 + outputMethod: last diff --git a/tests/loop-ext/value-from-loop.cwl b/tests/loop-ext/value-from-loop.cwl new file mode 100644 index 000000000..d7cd0a702 --- /dev/null +++ b/tests/loop-ext/value-from-loop.cwl @@ -0,0 +1,37 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.2 +class: Workflow +$namespaces: + cwltool: "http://commonwl.org/cwltool#" +requirements: + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + requirements: + cwltool:Loop: + loopWhen: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl index be5e6468e..04ec87a18 100644 --- a/tests/loop/all-output-loop-no-iteration.cwl +++ b/tests/loop/all-output-loop-no-iteration.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} inputs: @@ -24,9 +22,7 @@ steps: in: i1: i1 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 1) - loop: - i1: o1 - outputMethod: all + when: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: all diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl index 096be5192..157432e8f 100644 --- a/tests/loop/all-output-loop.cwl +++ b/tests/loop/all-output-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} inputs: @@ -24,9 +22,7 @@ steps: in: i1: i1 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: o1 - outputMethod: all + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: all diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl index 453a7613e..04c64f0dd 100644 --- a/tests/loop/default-value-loop.cwl +++ b/tests/loop/default-value-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -41,11 +39,9 @@ steps: in: i1: i1 out: [ o1 ] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 20) - loop: - i1: - loopSource: o1 - default: 5 - outputMethod: all \ No newline at end of file + when: $(inputs.i1 < 20) + loop: + i1: + outputSource: o1 + default: 5 + outputMethod: all diff --git a/tests/loop/invalid-loop-scatter.cwl b/tests/loop/invalid-loop-scatter.cwl index fa6fa248c..02451351d 100644 --- a/tests/loop/invalid-loop-scatter.cwl +++ b/tests/loop/invalid-loop-scatter.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -25,12 +23,10 @@ steps: o1: int expression: > ${return {'o1': inputs.i1 + inputs.i2};} - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-loop-when-exception.cwl b/tests/loop/invalid-loop-when-exception.cwl new file mode 100644 index 000000000..33f7795c4 --- /dev/null +++ b/tests/loop/invalid-loop-when-exception.cwl @@ -0,0 +1,36 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + when: | + ${ + throw new Error("failed"); + } + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/invalid-loop-when-exception2.cwl b/tests/loop/invalid-loop-when-exception2.cwl new file mode 100644 index 000000000..8e80250c4 --- /dev/null +++ b/tests/loop/invalid-loop-when-exception2.cwl @@ -0,0 +1,40 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + when: | + ${ + if (inputs.i1 != 1) { + throw new Error("failed"); + } else { + return true; + } + } + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl index c56ada500..811ba6c6e 100644 --- a/tests/loop/invalid-multi-source-loop-no-requirement.cwl +++ b/tests/loop/invalid-multi-source-loop-no-requirement.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -58,11 +56,9 @@ steps: in: i1: i1 out: [osmall, obig] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 20) - loop: - i1: - loopSource: [osmall, obig] - pickValue: the_only_non_null - outputMethod: all \ No newline at end of file + when: $(inputs.i1 < 20) + loop: + i1: + loopSource: [osmall, obig] + pickValue: the_only_non_null + outputMethod: all diff --git a/tests/loop/invalid-no-loopWhen.cwl b/tests/loop/invalid-no-loopWhen.cwl index c1f52f6f4..0a869be8e 100644 --- a/tests/loop/invalid-no-loopWhen.cwl +++ b/tests/loop/invalid-no-loopWhen.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -25,11 +23,9 @@ steps: o1: int expression: > ${return {'o1': inputs.i1 + inputs.i2};} - requirements: - cwltool:Loop: - loop: - i1: o1 - outputMethod: last + loop: + i1: o1 + outputMethod: last in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-non-boolean-loopWhen.cwl b/tests/loop/invalid-non-boolean-loopWhen.cwl index 99794d1e8..ea807ba3a 100644 --- a/tests/loop/invalid-non-boolean-loopWhen.cwl +++ b/tests/loop/invalid-non-boolean-loopWhen.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -25,12 +23,10 @@ steps: o1: int expression: > ${return {'o1': inputs.i1 + inputs.i2};} - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1) + loop: + i1: o1 + outputMethod: last in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-non-boolean-loopWhen2.cwl b/tests/loop/invalid-non-boolean-loopWhen2.cwl new file mode 100644 index 000000000..ba13e38ca --- /dev/null +++ b/tests/loop/invalid-non-boolean-loopWhen2.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + when: '$(inputs.i1 == 1 ? true : "I am a string")' + loop: + i1: o1 + outputMethod: last + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl index d2d7668ad..c232194bc 100644 --- a/tests/loop/invalid-value-from-loop-no-requirement.cwl +++ b/tests/loop/invalid-value-from-loop-no-requirement.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} inputs: @@ -27,10 +25,8 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: - valueFrom: $(inputs.i1 + 1) - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl index f62ea177c..bc805c2f5 100644 --- a/tests/loop/loop-inside-loop-all.cwl +++ b/tests/loop/loop-inside-loop-all.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -45,20 +43,16 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 <= inputs.i2) - loop: - i1: o1 - outputMethod: all + when: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: all in: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i2 < 4) - loop: - i2: - valueFrom: $(inputs.i2 + 1) - outputMethod: all + when: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl index 223fe542b..9b9fd7367 100644 --- a/tests/loop/loop-inside-loop.cwl +++ b/tests/loop/loop-inside-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -41,20 +39,16 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 <= inputs.i2) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: last in: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i2 < 4) - loop: - i2: - valueFrom: $(inputs.i2 + 1) - outputMethod: all + when: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl index 358309e23..3fc7fe257 100644 --- a/tests/loop/loop-inside-scatter.cwl +++ b/tests/loop/loop-inside-scatter.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -40,12 +38,10 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last in: i1: i1 i2: i2 diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl index 7f72db224..8c2a271cf 100644 --- a/tests/loop/multi-source-loop.cwl +++ b/tests/loop/multi-source-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} MultipleInputFeatureRequirement: {} @@ -59,11 +57,9 @@ steps: in: i1: i1 out: [osmall, obig] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 20) - loop: - i1: - loopSource: [osmall, obig] - pickValue: the_only_non_null - outputMethod: all \ No newline at end of file + when: $(inputs.i1 < 20) + loop: + i1: + outputSource: [osmall, obig] + pickValue: the_only_non_null + outputMethod: all diff --git a/tests/loop/opt-var-loop.cwl b/tests/loop/opt-var-loop.cwl new file mode 100644 index 000000000..d74f7c225 --- /dev/null +++ b/tests/loop/opt-var-loop.cwl @@ -0,0 +1,30 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + run: + class: ExpressionTool + inputs: + i1: int + i2: int? + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop/scatter-inside-loop.cwl b/tests/loop/scatter-inside-loop.cwl index 7a4c394be..ab857b0d0 100644 --- a/tests/loop/scatter-inside-loop.cwl +++ b/tests/loop/scatter-inside-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} ScatterFeatureRequirement: {} @@ -45,9 +43,7 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1[0] < 10) - loop: - i1: o1 - outputMethod: last \ No newline at end of file + when: $(inputs.i1[0] < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl index 451083a4b..a5678e735 100644 --- a/tests/loop/single-var-loop-no-iteration.cwl +++ b/tests/loop/single-var-loop-no-iteration.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} @@ -25,9 +23,7 @@ steps: in: i1: i1 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 1) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl index 0feac136d..ef3819ee1 100644 --- a/tests/loop/single-var-loop.cwl +++ b/tests/loop/single-var-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} @@ -25,9 +23,7 @@ steps: in: i1: i1 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl index bcc589215..7e50afe02 100644 --- a/tests/loop/two-vars-loop-2.cwl +++ b/tests/loop/two-vars-loop-2.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} inputs: @@ -27,9 +25,7 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: o1 - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl index 2a16ae7db..15b659e4a 100644 --- a/tests/loop/two-vars-loop.cwl +++ b/tests/loop/two-vars-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} inputs: @@ -28,10 +26,8 @@ steps: i1: i1 i2: i2 out: [o1, o2] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: o1 - i2: o2 - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: o1 + i2: o2 + outputMethod: last diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl index d7cd0a702..545029bd1 100644 --- a/tests/loop/value-from-loop.cwl +++ b/tests/loop/value-from-loop.cwl @@ -1,8 +1,6 @@ #!/usr/bin/env cwl-runner -cwlVersion: v1.2 +cwlVersion: v1.3.0-dev1 class: Workflow -$namespaces: - cwltool: "http://commonwl.org/cwltool#" requirements: InlineJavascriptRequirement: {} StepInputExpressionRequirement: {} @@ -28,10 +26,8 @@ steps: i1: i1 i2: i2 out: [o1] - requirements: - cwltool:Loop: - loopWhen: $(inputs.i1 < 10) - loop: - i1: - valueFrom: $(inputs.i1 + 1) - outputMethod: last + when: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last diff --git a/tests/test_loop.py b/tests/test_loop.py index ab1d9b0f6..269251627 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,4 +1,4 @@ -"""Test the prototype loop extension.""" +"""Test the 1.3 loop feature.""" import json from io import StringIO @@ -10,9 +10,9 @@ def test_validate_loop() -> None: - """Affirm that a loop workflow validates with --enable-ext.""" + """Affirm that a loop workflow validates with --enable-dev.""" params = [ - "--enable-ext", + "--enable-dev", "--validate", get_data("tests/loop/single-var-loop.cwl"), ] @@ -20,7 +20,7 @@ def test_validate_loop() -> None: def test_validate_loop_fail_no_ext() -> None: - """Affirm that a loop workflow does not validate when --enable-ext is missing.""" + """Affirm that a loop workflow does not validate when --enable-dev is missing.""" params = [ "--validate", get_data("tests/loop/single-var-loop.cwl"), @@ -31,78 +31,60 @@ def test_validate_loop_fail_no_ext() -> None: def test_validate_loop_fail_scatter() -> None: """Affirm that a loop workflow does not validate if scatter and loop directives are on the same step.""" params = [ - "--enable-ext", + "--enable-dev", "--validate", get_data("tests/loop/invalid-loop-scatter.cwl"), ] assert main(params) == 1 -def test_validate_loop_fail_when() -> None: - """Affirm that a loop workflow does not validate if when and loop directives are on the same step.""" - params = [ - "--enable-ext", - "--validate", - get_data("tests/loop/invalid-loop-when.cwl"), - ] - assert main(params) == 1 - - def test_validate_loop_fail_no_loop_when() -> None: - """Affirm that a loop workflow does not validate if no loopWhen directive is specified.""" + """Affirm that a loop workflow does not validate if no 'when' directive is specified.""" params = [ - "--enable-ext", + "--enable-dev", "--validate", get_data("tests/loop/invalid-no-loopWhen.cwl"), ] assert main(params) == 1 -def test_validate_loop_fail_on_workflow() -> None: - """Affirm that a workflow does not validate if it contains a Loop requirement.""" - params = [ - "--enable-ext", - "--validate", - get_data("tests/loop/invalid-loop-workflow.cwl"), - ] - assert main(params) == 1 - - -def test_validate_loop_fail_on_command_line_tool() -> None: - """Affirm that a CommandLineTool does not validate if it contains a Loop requirement.""" +def test_loop_fail_loop_when_exception() -> None: + """Affirm that a loop workflow fails if 'when' directive throws an exception.""" params = [ - "--enable-ext", - "--validate", - get_data("tests/loop/invalid-loop-command-line-tool.cwl"), + "--enable-dev", + get_data("tests/loop/invalid-loop-when-exception.cwl"), + get_data("tests/loop/two-vars-loop-job.yml"), ] assert main(params) == 1 -def test_validate_loop_fail_on_expression_tool() -> None: - """Affirm that an ExpressionTool does not validate if it contains a Loop requirement.""" +def test_loop_fail_loop_when_exception_second_iteration() -> None: + """Affirm that a loop workflow fails if when directive throws an + exception on second iteration.""" params = [ - "--enable-ext", - "--validate", - get_data("tests/loop/invalid-loop-expression-tool.cwl"), + "--enable-dev", + get_data("tests/loop/invalid-loop-when-exception2.cwl"), + get_data("tests/loop/two-vars-loop-job.yml"), ] assert main(params) == 1 -def test_validate_loop_fail_on_hint() -> None: - """Affirm that a loop workflow does not validate if it contains a Loop hint.""" +def test_loop_fail_non_boolean_loop_when() -> None: + """Affirm that a loop workflow fails if 'when' directive returns a non-boolean value.""" params = [ - "--enable-ext", - "--validate", - get_data("tests/loop/invalid-loop-hint.cwl"), + "--enable-dev", + get_data("tests/loop/invalid-non-boolean-loopWhen.cwl"), + get_data("tests/loop/two-vars-loop-job.yml"), ] assert main(params) == 1 -def test_loop_fail_non_boolean_loop_when() -> None: - """Affirm that a loop workflow fails if loopWhen directive returns a non-boolean value.""" +def test_loop_fail_non_boolean_loop_second_when() -> None: + """Affirm that a loop workflow fails if 'when' directive returns + a non-boolean value on the second iteration.""" params = [ - "--enable-ext", - get_data("tests/loop/invalid-non-boolean-loopWhen.cwl"), + "--enable-dev", + get_data("tests/loop/invalid-non-boolean-loopWhen2.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] assert main(params) == 1 @@ -112,7 +94,7 @@ def test_loop_single_variable() -> None: """Test a simple loop case with a single variable.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/single-var-loop.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] @@ -122,10 +104,10 @@ def test_loop_single_variable() -> None: def test_loop_single_variable_no_iteration() -> None: - """Test a simple loop case with a single variable and a false loopWhen condition.""" + """Test a simple loop case with a single variable and a false 'when' condition.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/single-var-loop-no-iteration.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] @@ -138,7 +120,7 @@ def test_loop_two_variables() -> None: """Test a loop case with two variables, which are both back-propagated between iterations.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/two-vars-loop.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] @@ -151,7 +133,7 @@ def test_loop_two_variables_single_backpropagation() -> None: """Test loop with 2 variables, but when only one of them is back-propagated between iterations.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/two-vars-loop-2.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] @@ -164,7 +146,7 @@ def test_loop_with_all_output_method() -> None: """Test a loop case with outputMethod set to all.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/all-output-loop.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] @@ -174,10 +156,10 @@ def test_loop_with_all_output_method() -> None: def test_loop_with_all_output_method_no_iteration() -> None: - """Test a loop case with outputMethod set to all and a false loopWhen condition.""" + """Test a loop case with outputMethod set to all and a false 'when' condition.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/all-output-loop-no-iteration.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] @@ -190,7 +172,7 @@ def test_loop_value_from() -> None: """Test a loop case with a variable generated by a valueFrom directive.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/value-from-loop.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] @@ -202,7 +184,7 @@ def test_loop_value_from() -> None: def test_loop_value_from_fail_no_requirement() -> None: """Test workflow loop fails for valueFrom without StepInputExpressionRequirement.""" params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/invalid-value-from-loop-no-requirement.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] @@ -213,7 +195,7 @@ def test_loop_inside_scatter() -> None: """Test a loop subworkflow inside a scatter step.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/loop-inside-scatter.cwl"), get_data("tests/loop/loop-inside-scatter-job.yml"), ] @@ -226,7 +208,7 @@ def test_scatter_inside_loop() -> None: """Test a loop workflow with inside a scatter step.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/scatter-inside-loop.cwl"), get_data("tests/loop/loop-inside-scatter-job.yml"), ] @@ -235,11 +217,24 @@ def test_scatter_inside_loop() -> None: assert json.loads(stream.getvalue()) == expected +def test_loop_opt_variable() -> None: + """Test a loop case with two variables but one is optional.""" + stream = StringIO() + params = [ + "--enable-dev", + get_data("tests/loop/opt-var-loop.cwl"), + get_data("tests/loop/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": 10} + assert json.loads(stream.getvalue()) == expected + + def test_nested_loops() -> None: """Test a workflow with two nested loops.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/loop-inside-loop.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] @@ -252,7 +247,7 @@ def test_nested_loops_all() -> None: """Test a workflow with two nested loops, both with outputMethod set to all.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/loop-inside-loop-all.cwl"), get_data("tests/loop/two-vars-loop-job.yml"), ] @@ -265,7 +260,7 @@ def test_multi_source_loop_input() -> None: """Test a loop with two sources, which are selected through a pickValue directive.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/multi-source-loop.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] @@ -277,7 +272,7 @@ def test_multi_source_loop_input() -> None: def test_multi_source_loop_input_fail_no_requirement() -> None: """Test that a loop with two sources fails without MultipleInputFeatureRequirement.""" params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/invalid-multi-source-loop-no-requirement.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] @@ -288,7 +283,7 @@ def test_default_value_loop() -> None: """Test a loop whose source has a default value.""" stream = StringIO() params = [ - "--enable-ext", + "--enable-dev", get_data("tests/loop/default-value-loop.cwl"), get_data("tests/loop/single-var-loop-job.yml"), ] diff --git a/tests/test_loop_ext.py b/tests/test_loop_ext.py new file mode 100644 index 000000000..499dd17b4 --- /dev/null +++ b/tests/test_loop_ext.py @@ -0,0 +1,331 @@ +"""Test the prototype cwltool:Loop extension.""" + +import json +from io import StringIO +from typing import MutableMapping, MutableSequence + +from cwltool.main import main + +from .util import get_data + + +def test_validate_loop() -> None: + """Affirm that a loop workflow validates with --enable-ext.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/single-var-loop.cwl"), + ] + assert main(params) == 0 + + +def test_validate_loop_fail_no_ext() -> None: + """Affirm that a loop workflow does not validate when --enable-ext is missing.""" + params = [ + "--validate", + get_data("tests/loop-ext/single-var-loop.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_scatter() -> None: + """Affirm that a loop workflow does not validate if scatter and loop directives are on the same step.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-loop-scatter.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_when() -> None: + """Affirm that a loop workflow does not validate if when and loop directives are on the same step.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-loop-when.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_no_loop_when() -> None: + """Affirm that a loop workflow does not validate if no loopWhen directive is specified.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-no-loopWhen.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_on_workflow() -> None: + """Affirm that a workflow does not validate if it contains a Loop requirement.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-loop-workflow.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_on_command_line_tool() -> None: + """Affirm that a CommandLineTool does not validate if it contains a Loop requirement.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-loop-command-line-tool.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_on_expression_tool() -> None: + """Affirm that an ExpressionTool does not validate if it contains a Loop requirement.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-loop-expression-tool.cwl"), + ] + assert main(params) == 1 + + +def test_validate_loop_fail_on_hint() -> None: + """Affirm that a loop workflow does not validate if it contains a Loop hint.""" + params = [ + "--enable-ext", + "--validate", + get_data("tests/loop-ext/invalid-loop-hint.cwl"), + ] + assert main(params) == 1 + + +def test_loop_fail_loop_when_exception() -> None: + """Affirm that a loop workflow fails if loopWhen directive throws an exception.""" + params = [ + "--enable-ext", + get_data("tests/loop-ext/invalid-loop-when-exception.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + assert main(params) == 1 + + +def test_loop_fail_non_boolean_loop_when() -> None: + """Affirm that a loop workflow fails if loopWhen directive returns a non-boolean value.""" + params = [ + "--enable-ext", + get_data("tests/loop-ext/invalid-non-boolean-loopWhen.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + assert main(params) == 1 + + +def test_loop_fail_non_boolean_loop_second_when() -> None: + """Affirm that a loop workflow fails if loopWhen directive returns + a non-boolean value on the second iteration.""" + params = [ + "--enable-ext", + get_data("tests/loop-ext/invalid-non-boolean-loopWhen2.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + assert main(params) == 1 + + +def test_loop_single_variable() -> None: + """Test a simple loop case with a single variable.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/single-var-loop.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": 10} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_single_variable_no_iteration() -> None: + """Test a simple loop case with a single variable and a false loopWhen condition.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/single-var-loop-no-iteration.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": None} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_two_variables() -> None: + """Test a loop case with two variables, which are both back-propagated between iterations.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/two-vars-loop.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": 10} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_two_variables_single_backpropagation() -> None: + """Test loop with 2 variables, but when only one of them is back-propagated between iterations.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/two-vars-loop-2.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": 10} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_with_all_output_method() -> None: + """Test a loop case with outputMethod set to all.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/all-output-loop.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [2, 3, 4, 5, 6, 7, 8, 9, 10]} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_with_all_output_method_no_iteration() -> None: + """Test a loop case with outputMethod set to all and a false loopWhen condition.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/all-output-loop-no-iteration.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected: MutableMapping[str, MutableSequence[int]] = {"o1": []} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_value_from() -> None: + """Test a loop case with a variable generated by a valueFrom directive.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/value-from-loop.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": 10} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_value_from_fail_no_requirement() -> None: + """Test workflow loop fails for valueFrom without StepInputExpressionRequirement.""" + params = [ + "--enable-ext", + get_data("tests/loop-ext/invalid-value-from-loop-no-requirement.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + assert main(params) == 1 + + +def test_loop_inside_scatter() -> None: + """Test a loop subworkflow inside a scatter step.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/loop-inside-scatter.cwl"), + get_data("tests/loop-ext/loop-inside-scatter-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [10, 10, 10, 10, 10]} + assert json.loads(stream.getvalue()) == expected + + +def test_scatter_inside_loop() -> None: + """Test a loop workflow with inside a scatter step.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/scatter-inside-loop.cwl"), + get_data("tests/loop-ext/loop-inside-scatter-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [10, 11, 12, 13, 14]} + assert json.loads(stream.getvalue()) == expected + + +def test_loop_opt_variable() -> None: + """Test a loop case with two variables but one is optional.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/opt-var-loop.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": 10} + assert json.loads(stream.getvalue()) == expected + + +def test_nested_loops() -> None: + """Test a workflow with two nested loops.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/loop-inside-loop.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [2, 3, 4]} + assert json.loads(stream.getvalue()) == expected + + +def test_nested_loops_all() -> None: + """Test a workflow with two nested loops, both with outputMethod set to all.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/loop-inside-loop-all.cwl"), + get_data("tests/loop-ext/two-vars-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [[2], [2, 3], [2, 3, 4]]} + assert json.loads(stream.getvalue()) == expected + + +def test_multi_source_loop_input() -> None: + """Test a loop with two sources, which are selected through a pickValue directive.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/multi-source-loop.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [2, 3, 4, 5, 8, 11, 14, 17, 20]} + assert json.loads(stream.getvalue()) == expected + + +def test_multi_source_loop_input_fail_no_requirement() -> None: + """Test that a loop with two sources fails without MultipleInputFeatureRequirement.""" + params = [ + "--enable-ext", + get_data("tests/loop-ext/invalid-multi-source-loop-no-requirement.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + assert main(params) == 1 + + +def test_default_value_loop() -> None: + """Test a loop whose source has a default value.""" + stream = StringIO() + params = [ + "--enable-ext", + get_data("tests/loop-ext/default-value-loop.cwl"), + get_data("tests/loop-ext/single-var-loop-job.yml"), + ] + main(params, stdout=stream) + expected = {"o1": [8, 11, 14, 17, 20]} + assert json.loads(stream.getvalue()) == expected