diff --git a/.github/config/wordlist.txt b/.github/config/wordlist.txt index ec05e6b..30d75d8 100644 --- a/.github/config/wordlist.txt +++ b/.github/config/wordlist.txt @@ -1,3 +1,4 @@ +abstractworkflowstep acyclic amstutz arrayschema @@ -160,6 +161,9 @@ loadlistingrequirement localhost lookahead loopback +loopinput +loopoutputmethod +loopworkflowstep maccallum macos mappredicate @@ -194,6 +198,7 @@ outputbinding outputenumschema outputeval outputformat +outputmethod outputschema outputsink outputsource @@ -231,6 +236,7 @@ saladversion sbgenomics scatterfeaturerequirement scattermethod +scatterworkflowstep schemadefrequirement schemaorg schemas diff --git a/.github/workflows/cwltool.yml b/.github/workflows/cwltool.yml index 9bb190a..f634623 100644 --- a/.github/workflows/cwltool.yml +++ b/.github/workflows/cwltool.yml @@ -24,7 +24,7 @@ jobs: python-version: '3.9.x' - name: Setup prerequirements - run: pip install "cwltest>=2.3" pytest-xdist "git+https://github.com/common-workflow-language/cwltool.git@main#egg=cwltool" + run: pip install "cwltest>=2.3" pytest-xdist "cwltool>=3.1.20240909164951" - name: Copy in cwltool-specific configuration for the cwltest pytest plugin run: cp "$(python -c 'from cwltool.tests.util import get_data; print(get_data("tests/cwl-conformance/cwltool-conftest.py"))')" conftest.py @@ -33,4 +33,4 @@ jobs: run: cp conformance_tests.yaml conformance_tests.cwltest.yaml - name: Run tests against the reference runner - run: python -m pytest conformance_tests.cwltest.yaml -n auto -rs + run: python -m pytest conformance_tests.cwltest.yaml -n auto -rsfE diff --git a/CONFORMANCE_TESTS.md b/CONFORMANCE_TESTS.md index 2262870..8e401ac 100644 --- a/CONFORMANCE_TESTS.md +++ b/CONFORMANCE_TESTS.md @@ -144,7 +144,7 @@ We will use this single entry to explain the format Must include one or more of the following tags: `command_line_tool`, `expression_tool` or `workflow`. If the test does not test any optional features, the tag `required` is required. -Because `conformance_tests.yaml` is a `schema-salad` processed document, [`$import`](https://www.commonwl.org/v1.2/SchemaSalad.html#Import) +Because `conformance_tests.yaml` is a `schema-salad` processed document, [`$import`](https://www.commonwl.org/v1.3/SchemaSalad.html#Import) can be used to organize the tests into separate files. Currently, the main file is too big (over 3400 lines); we are slowly re-organizing it. @@ -166,7 +166,7 @@ At any level, if there is an extra field, then that will be considered an error. An exception to this is `class: File` and `class: Directory` objects, the `cwl-runner` under test can add additional fields here without causing a test to fail. Likewise, if you don't want to test some aspect of a `class: File` or `class: Directory` object (like `nameext`) you can just omit it. -[According to the CWL standards](https://www.commonwl.org/v1.2/CommandLineTool.html#File), the format of the `location` field in +[According to the CWL standards](https://www.commonwl.org/v1.3/CommandLineTool.html#File), the format of the `location` field in `class: File` and `class: Directory` is implementation specific and we should not be testing them. Please remember to use `location: Any` for them. diff --git a/Workflow.yml b/Workflow.yml index 176de9a..6c20b8c 100644 --- a/Workflow.yml +++ b/Workflow.yml @@ -40,6 +40,12 @@ $graph: CWL group. ## Changelog + + * Added a new Loop feature to support iterative workflows. From a technical perspective, this led to the + introduction of a `LoopWorkflowStep`[#LoopWorkflowStep] element to represent iterative subworkflows, a + `LoopInput`[#LoopInput] object to handle termination conditions, and a `LoopOutputMethod`[#LoopOutputMethod] + feature to manage loop output data history. + See also the [CWL Command Line Tool Description, v1.3.0-dev1 changelog](CommandLineTool.html#Changelog). For other changes since CWL v1.0, see the [CWL Workflow Description, v1.1 changelog](https://www.commonwl.org/v1.1/Workflow.html#Changelog) @@ -275,7 +281,7 @@ $graph: - type: record name: WorkflowStepInput extends: [Identified, InputSink, LoadContents, Labeled] - docParent: "#WorkflowStep" + docParent: "#AbstractWorkflowStep" doc: | The input of a workflow step connects an upstream parameter (from the workflow inputs, or the outputs of other workflows steps) with the input @@ -301,7 +307,7 @@ $graph: in the workflow or workflow step requirements. If the sink parameter is an array, or named in a [workflow - scatter](#WorkflowStep) operation, there may be multiple inbound + scatter](#ScatterWorkflowStep) operation, there may be multiple inbound data links listed in the `source` field. The values from the input links are merged depending on the method specified in the `linkMerge` field. If both `linkMerge` and `pickValue` are null @@ -338,7 +344,7 @@ $graph: 3. Before `scatter` or `valueFrom`. This is specifically intended to be useful in combination with - [conditional execution](#WorkflowStep), where several upstream + [conditional execution](#AbstractWorkflowStep), where several upstream steps may be connected to a single input (`source` is a list), and skipped steps produce null values. @@ -430,7 +436,7 @@ $graph: - type: record name: WorkflowStepOutput - docParent: "#WorkflowStep" + docParent: "#AbstractWorkflowStep" extends: Identified doc: | Associate an output parameter of the underlying process with a workflow @@ -443,19 +449,68 @@ $graph: to connect the output value to downstream parameters. +- name: LoopInput + type: record + extends: [Identified, OutputSink] + fields: + - name: default + type: ["null", File, Directory, Any] + doc: | + The default value for this parameter to use if either there is no + `outputSource` field, or the value produced by the `source` is `null`. The + default must be applied prior to scattering or evaluating `valueFrom`. + jsonldPredicate: + _id: "sld:default" + _container: "@list" + noLinkCheck: true + - name: valueFrom + type: + - "null" + - string + - Expression + jsonldPredicate: "cwl:valueFrom" + doc: | + To use valueFrom, [StepInputExpressionRequirement](#StepInputExpressionRequirement) must + be specified in the workflow or workflow step requirements. + + If `valueFrom` is a constant string value, use this as the value for + this input parameter. + + If `valueFrom` is a parameter reference or expression, it must be + evaluated to yield the actual value to be assigned to the input field. + + The `self` value in the parameter reference or expression must be + `null` if there is no `source` field, or the value of the + parameter(s) specified in the `source` field. + + The value of `inputs` in the parameter reference or expression must be + the input object to the previous iteration of the workflow step (or the initial + inputs for the first iteration). + + - name: ScatterMethod type: enum - docParent: "#WorkflowStep" - doc: The scatter method, as described in [workflow step scatter](#WorkflowStep). + docParent: "#ScatterWorkflowStep" + doc: The scatter method, as described in [workflow step scatter](#ScatterWorkflowStep). symbols: - dotproduct - nested_crossproduct - flat_crossproduct -- name: WorkflowStep +- name: LoopOutputMethod + type: enum + docParent: "#LoopWorkflowStep" + doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep). + symbols: + - last_iteration + - all_iterations + + +- name: AbstractWorkflowStep type: record extends: [Identified, Labeled, sld:Documented] + abstract: true docParent: "#Workflow" doc: | A workflow step is an executable element of a workflow. It specifies the @@ -463,71 +518,23 @@ $graph: `Workflow`) in the `run` field and connects the input and output parameters of the underlying process to workflow parameters. - # Scatter/gather - - To use scatter/gather, - [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified - in the workflow or workflow step requirements. - - A "scatter" operation specifies that the associated workflow step or - subworkflow should execute separately over a list of input elements. Each - job making up a scatter operation is independent and may be executed - concurrently. - - The `scatter` field specifies one or more input parameters which will be - scattered. An input parameter may be listed more than once. The declared - type of each input parameter implicitly becomes an array of items of the - input parameter type. If a parameter is listed more than once, it becomes - a nested array. As a result, upstream parameters which are connected to - scattered parameters must be arrays. - - All output parameter types are also implicitly wrapped in arrays. Each job - in the scatter results in an entry in the output array. - - If any scattered parameter runtime value is an empty array, all outputs are - set to empty arrays and no work is done for the step, according to - applicable scattering rules. - - If `scatter` declares more than one input parameter, `scatterMethod` - describes how to decompose the input into a discrete set of jobs. - - * **dotproduct** specifies that each of the input arrays are aligned and one - element taken from each array to construct each job. It is an error - if all input arrays are not the same length. - - * **nested_crossproduct** specifies the Cartesian product of the inputs, - producing a job for every combination of the scattered inputs. The - output must be nested arrays for each level of scattering, in the - order that the input arrays are listed in the `scatter` field. - - * **flat_crossproduct** specifies the Cartesian product of the inputs, - producing a job for every combination of the scattered inputs. The - output arrays must be flattened to a single level, but otherwise listed in the - order that the input arrays are listed in the `scatter` field. - # Conditional execution (Optional) Conditional execution makes execution of a step conditional on an expression. A step that is not executed is "skipped". A skipped step produces `null` for all output parameters. - The condition is evaluated after `scatter`, using the input object - of each individual scatter job. This means over a set of scatter - jobs, some may be executed and some may be skipped. When the - results are gathered, skipped steps must be `null` in the output - arrays. - The `when` field controls conditional execution. This is an expression that must be evaluated with `inputs` bound to the step input object (or individual scatter job), and returns a boolean value. It is an error if this expression returns a value other than `true` or `false`. - Conditionals in CWL are an optional feature and are not required - to be implemented by all consumers of CWL documents. An - implementation that does not support conditionals must return a - fatal error when attempting to execute a workflow that uses - conditional constructs the implementation does not support. + Conditional execution in CWL is an optional feature and is not required + to be implemented by all consumers of CWL documents. An implementation that + does not support conditional executions must return a fatal error when + attempting to execute a workflow that uses conditional constructs the + implementation does not support. # Subworkflows @@ -603,6 +610,68 @@ $graph: If defined, only run the step when the expression evaluates to `true`. If `false` the step is skipped. A skipped step produces a `null` on each output. + + +- name: WorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + + +- name: ScatterWorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + doc: | + To use scatter/gather, + [ScatterFeatureRequirement](#ScatterFeatureRequirement) must be specified + in the workflow or workflow step requirements. + + A "scatter" operation specifies that the associated workflow step or + subworkflow should execute separately over a list of input elements. Each + job making up a scatter operation is independent and may be executed + concurrently. + + The `scatter` field specifies one or more input parameters which will be + scattered. An input parameter may be listed more than once. The declared + type of each input parameter implicitly becomes an array of items of the + input parameter type. If a parameter is listed more than once, it becomes + a nested array. As a result, upstream parameters which are connected to + scattered parameters must be arrays. + + All output parameter types are also implicitly wrapped in arrays. Each job + in the scatter results in an entry in the output array. + + If any scattered parameter runtime value is an empty array, all outputs are + set to empty arrays and no work is done for the step, according to + applicable scattering rules. + + If `scatter` declares more than one input parameter, `scatterMethod` + describes how to decompose the input into a discrete set of jobs. + + * **dotproduct** specifies that each of the input arrays are aligned and one + element taken from each array to construct each job. It is an error + if all input arrays are not the same length. + + * **nested_crossproduct** specifies the Cartesian product of the inputs, + producing a job for every combination of the scattered inputs. The + output must be nested arrays for each level of scattering, in the + order that the input arrays are listed in the `scatter` field. + + * **flat_crossproduct** specifies the Cartesian product of the inputs, + producing a job for every combination of the scattered inputs. The + output arrays must be flattened to a single level, but otherwise listed in the + order that the input arrays are listed in the `scatter` field. + + # Conditional execution (Optional) + + The condition is evaluated after `scatter`, using the input object + of each individual scatter job. This means over a set of scatter + jobs, some may be executed and some may be skipped. When the + results are gathered, skipped steps must be `null` in the output + arrays. + + fields: - name: scatter type: - string? @@ -621,6 +690,74 @@ $graph: "_type": "@vocab" +- name: LoopWorkflowStep + type: record + extends: AbstractWorkflowStep + docParent: "#Workflow" + doc: | + # Iterative execution (Optional) + + The `loop` field controls iterative execution. It defines the input + parameters of the loop iterations after the first one (inputs of the + first iteration are the step input parameters, as usual). If no + `loop` rule is specified for a given step `in` field, the initial + value is kept constant among all iterations. + + When a `loop` field is present, the `when` field is mandatory. It is + evaluated before each loop iteration and acts as a termination condition: + as soon as the `when` expression evaluates to `false`, the loop terminates + and the step outputs are propagated to the subsequent workflow steps. + + The `outputMethod` field describes how to deal with loop outputs after + termination: + + * **last_iteration** specifies that only the last computed element for + each output parameter should be propagated to the subsequent steps. + This is the default value. + + * **all_iterations** specifies that an array with all output values + computed at the end of each loop iteration should be propagated to + the subsequent steps. Elements in the array must be ordered according + to the loop iterations that produced them. + + Iterative execution in CWL is an optional feature and is not required + to be implemented by all consumers of CWL documents. An implementation that + does not support iterative executions must return a fatal error when + attempting to execute a workflow that uses iterative constructs the + implementation does not support. + + fields: + - name: loop + doc: | + Defines the input parameters of the loop iterations after the first one + (inputs of the first iteration are the step input parameters). If no + `loop` rule is specified for a given step `in` field, the initial value + is kept constant among all iterations. + type: LoopInput[]? + jsonldPredicate: + _id: "cwl:loop" + mapSubject: id + mapPredicate: outputSource + - name: outputMethod + doc: | + If not specified, the default method is "last_iteration". + type: LoopOutputMethod? + default: last_iteration + jsonldPredicate: + "_id": "cwl:outputMethod" + "_type": "@vocab" + - name: when + type: + - Expression + jsonldPredicate: "cwl:when" + doc: | + Only run the next iteration when the expression evaluates to `true`. + If the first iteration evaluates to `false` the step is skipped. + A skipped step produces a `null` on each output if the `outputMethod` + is set to `last_iteration`, and an empty array if the `outputMethod` + is set to `all_iterations`. + + - name: Workflow type: record extends: "#Process" @@ -701,7 +838,7 @@ $graph: concurrently, provided that dependencies between steps are met. type: - type: array - items: "#WorkflowStep" + items: AbstractWorkflowStep jsonldPredicate: mapSubject: id @@ -711,7 +848,7 @@ $graph: extends: ProcessRequirement doc: | Indicates that the workflow platform must support nested workflows in - the `run` field of [WorkflowStep](#WorkflowStep). + the `run` field of [AbstractWorkflowStep](#AbstractWorkflowStep). fields: - name: "class" type: @@ -729,7 +866,7 @@ $graph: extends: ProcessRequirement doc: | Indicates that the workflow platform must support the `scatter` and - `scatterMethod` fields of [WorkflowStep](#WorkflowStep). + `scatterMethod` fields of [ScatterWorkflowStep](#ScatterWorkflowStep). fields: - name: "class" type: diff --git a/conformance_tests.yaml b/conformance_tests.yaml index 8d397d7..ee8c0a6 100644 --- a/conformance_tests.yaml +++ b/conformance_tests.yaml @@ -3525,3 +3525,5 @@ an_array_of_mixed_booleans: [ false, true, false ] an_array_of_trues: [ true, true, true ] an_int: 42 + +- $import: tests/loop/test-index.yaml diff --git a/design-documents/conditionals-2019.md b/design-documents/conditionals-2019.md index 34f7d69..ca581c3 100644 --- a/design-documents/conditionals-2019.md +++ b/design-documents/conditionals-2019.md @@ -10,7 +10,7 @@ This is a documentation of the design and design decisions for conditionals as o ![dual scatter nested](conditionals/conditional-patterns-3.png) ![dual scatter flattened](conditionals/conditional-patterns-4.png) -The design adds a new field `when` to a `WorkflowStep`. This field is an expression that +The design adds a new field `when` to a `AbstractWorkflowStep`. This field is an expression that evaluates to `True` or `False`. The executor runs the step if the value is `True`, skips it if `False`. A skipped step produces `null` values on all its outputs. @@ -114,7 +114,7 @@ outputs: pickValue: first_non_null ``` -The new syntax adds a single field to `WorkflowStep` (`when`) and a new +The new syntax adds a single field to `AbstractWorkflowStep` (`when`) and a new operator called `pickValue` to the `WorkflowStepInput` and `WorkflowOutputParameter`. This is a fairly non-intrusive modification, fully backwards compatible (it's an addition, not a modification) and allows diff --git a/requirements.txt b/requirements.txt index f5c7cc1..5d99a3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -cwltool +cwltool \ No newline at end of file diff --git a/tests/conditionals/test-index.yaml b/tests/conditionals/test-index.yaml index cee57e9..f646a56 100644 --- a/tests/conditionals/test-index.yaml +++ b/tests/conditionals/test-index.yaml @@ -20,7 +20,7 @@ tags: [ conditional, inline_javascript, workflow ] - id: direct_required - doc: Conditional using intermediate WorkflowStep.in input + doc: Conditional using intermediate AbstractWorkflowStep.in input tool: cond-wf-002.cwl job: val.1.job.yaml output: @@ -186,7 +186,7 @@ out1: [ "foo 2", "foo 4", "foo 6", "bar 1", "bar 3", "bar 5"] - tags: [ conditional, inline_javascript, scatter, multiple, workflow ] + tags: [ conditional, inline_javascript, scatter, multiple_input, workflow ] - id: direct_optional_null_result_nojs doc: simplest conditional pattern (true), no javascript @@ -205,7 +205,7 @@ tags: [ conditional, workflow ] - id: direct_required_nojs - doc: Conditional using intermediate WorkflowStep.in input; no javascript + doc: Conditional using intermediate AbstractWorkflowStep.in input; no javascript tool: cond-wf-002_nojs.cwl job: val.1.job.yaml output: @@ -371,7 +371,7 @@ out1: [ "foo 2", "foo 4", "foo 6", "bar 1", "bar 3", "bar 5"] - tags: [ conditional, scatter, multiple, workflow ] + tags: [ conditional, scatter, multiple_input, workflow ] - id: cond-with-defaults-1 @@ -395,7 +395,7 @@ "size": 34 } ] - tags: [ conditional, scatter, multiple, workflow ] + tags: [ conditional, scatter, multiple_input, workflow ] - id: cond-with-defaults-2 doc: "Default inputs, choose step to run based on what was provided, second case" @@ -411,4 +411,4 @@ "size": 12 } ] - tags: [ conditional, scatter, multiple, workflow ] + tags: [ conditional, scatter, multiple_input, workflow ] diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl new file mode 100644 index 0000000..97d6240 --- /dev/null +++ b/tests/loop/all-output-loop-no-iteration.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: all_iterations + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl new file mode 100644 index 0000000..7001731 --- /dev/null +++ b/tests/loop/all-output-loop.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: all_iterations + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl new file mode 100644 index 0000000..fb5e00e --- /dev/null +++ b/tests/loop/default-value-loop.cwl @@ -0,0 +1,47 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: loop/o1 + pickValue: all_non_null +steps: + loop: + when: $(inputs.i1 < 20) + loop: + i1: + outputSource: o1 + default: 5 + outputMethod: all_iterations + run: + class: Workflow + inputs: + i1: int + outputs: + o1: + type: int? + outputSource: big_values/o1 + steps: + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [ o1 ] diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl new file mode 100644 index 0000000..aa83257 --- /dev/null +++ b/tests/loop/invalid-multi-source-loop-no-requirement.cwl @@ -0,0 +1,64 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened + pickValue: all_non_null +steps: + loop: + when: $(inputs.i1 < 20) + loop: + i1: + outputSource: [ osmall, obig ] + pickValue: the_only_non_null + outputMethod: all_iterations + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl new file mode 100644 index 0000000..52edf39 --- /dev/null +++ b/tests/loop/invalid-value-from-loop-no-requirement.cwl @@ -0,0 +1,32 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl new file mode 100644 index 0000000..3c2da4b --- /dev/null +++ b/tests/loop/loop-inside-loop-all.cwl @@ -0,0 +1,58 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: + type: array + items: + type: array + items: int + outputSource: loop1/o1 +steps: + loop1: + when: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all_iterations + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int[] + outputSource: loop2/o1 + steps: + loop2: + when: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: all_iterations + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + i2: i2 + out: [o1] + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl new file mode 100644 index 0000000..2681eb5 --- /dev/null +++ b/tests/loop/loop-inside-loop.cwl @@ -0,0 +1,54 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + StepInputExpressionRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int[] + outputSource: loop1/o1 +steps: + loop1: + when: $(inputs.i2 < 4) + loop: + i2: + valueFrom: $(inputs.i2 + 1) + outputMethod: all_iterations + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int + outputSource: loop2/o1 + steps: + loop2: + when: $(inputs.i1 <= inputs.i2) + loop: + i1: o1 + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + i2: i2 + out: [o1] + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/loop-inside-scatter-job.yml b/tests/loop/loop-inside-scatter-job.yml new file mode 100644 index 0000000..0d95d06 --- /dev/null +++ b/tests/loop/loop-inside-scatter-job.yml @@ -0,0 +1,2 @@ +i1: [1, 2, 3, 4, 5] +i2: 1 \ No newline at end of file diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl new file mode 100644 index 0000000..7c6db45 --- /dev/null +++ b/tests/loop/loop-inside-scatter.cwl @@ -0,0 +1,49 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int[] + i2: int +outputs: + o1: + type: int[] + outputSource: scatter/o1 +steps: + scatter: + run: + class: Workflow + inputs: + i1: int + i2: int + outputs: + o1: + type: int + outputSource: subworkflow/o1 + steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] + in: + i1: i1 + i2: i2 + scatter: i1 + out: [o1] diff --git a/tests/loop/multi-source-loop-no-pick-value.cwl b/tests/loop/multi-source-loop-no-pick-value.cwl new file mode 100644 index 0000000..7587681 --- /dev/null +++ b/tests/loop/multi-source-loop-no-pick-value.cwl @@ -0,0 +1,64 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + MultipleInputFeatureRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened +steps: + loop: + when: $(inputs.i1 < 5) + loop: + i1: + outputSource: [ osmall, obig ] + valueFrom: $(self[0]) + outputMethod: all_iterations + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl new file mode 100644 index 0000000..af0adc8 --- /dev/null +++ b/tests/loop/multi-source-loop.cwl @@ -0,0 +1,65 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + MultipleInputFeatureRequirement: {} + ScatterFeatureRequirement: {} + SubworkflowFeatureRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int[] + outputSource: [loop/osmall, loop/obig] + linkMerge: merge_flattened + pickValue: all_non_null +steps: + loop: + when: $(inputs.i1 < 20) + loop: + i1: + outputSource: [ osmall, obig ] + pickValue: the_only_non_null + outputMethod: all_iterations + run: + class: Workflow + inputs: + i1: int + outputs: + osmall: + type: int? + outputSource: small_values/o1 + obig: + type: int? + outputSource: big_values/o1 + steps: + small_values: + when: $(inputs.i1 < 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] + big_values: + when: $(inputs.i1 >= 5) + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 3};} + in: + i1: i1 + out: [ o1 ] + in: + i1: i1 + out: [osmall, obig] diff --git a/tests/loop/single-var-loop-job.yml b/tests/loop/single-var-loop-job.yml new file mode 100644 index 0000000..d491f14 --- /dev/null +++ b/tests/loop/single-var-loop-job.yml @@ -0,0 +1 @@ +i1: 1 \ No newline at end of file diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl new file mode 100644 index 0000000..9dd60d7 --- /dev/null +++ b/tests/loop/single-var-loop-no-iteration.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 1) + loop: + i1: o1 + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl new file mode 100644 index 0000000..e94eb05 --- /dev/null +++ b/tests/loop/single-var-loop.cwl @@ -0,0 +1,28 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + 1};} + in: + i1: i1 + out: [o1] diff --git a/tests/loop/test-index.yaml b/tests/loop/test-index.yaml new file mode 100644 index 0000000..1122f11 --- /dev/null +++ b/tests/loop/test-index.yaml @@ -0,0 +1,109 @@ +- job: single-var-loop-job.yml + tool: single-var-loop.cwl + id: loop_single_variable + doc: "Test a simple loop case with a single variable" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: single-var-loop-job.yml + tool: single-var-loop-no-iteration.cwl + id: loop_single_variable_no_iteration + doc: "Test a simple loop case with a single variable and a false condition" + output: + o1: null + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: two-vars-loop.cwl + id: loop_two_variables + doc: "Test a loop case with two variables, which are both back-propagated between iterations" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: two-vars-loop-2.cwl + id: loop_two_variables_single_backpropagation + doc: "Test a loop case with two variables, but when only one of them is back-propagated between iterations" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: single-var-loop-job.yml + tool: all-output-loop.cwl + id: loop_with_all_output_method + doc: "Test a loop case with outputMethod set to all_iterations" + output: + o1: [2, 3, 4, 5, 6, 7, 8, 9, 10] + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: single-var-loop-job.yml + tool: all-output-loop-no-iteration.cwl + id: loop_with_all_output_method_no_iteration + doc: "Test a loop case with outputMethod set to all_iterations and a false condition" + output: + o1: [] + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: value-from-loop.cwl + id: loop_value_from + doc: "Test a loop case with a variable generated by a valueFrom directive" + output: + o1: 10 + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: two-vars-loop-job.yml + tool: invalid-value-from-loop-no-requirement.cwl + id: loop_value_from_fail_no_requirement + doc: "Test that a workflow loop fails if a valueFrom directive is specified without StepInputExpressionRequirement" + should_fail: true + tags: [ conditional, loop, inline_javascript, workflow ] + +- job: loop-inside-scatter-job.yml + tool: loop-inside-scatter.cwl + id: loop_inside_scatter + doc: "Test a loop subworkflow inside a scatter step" + output: + o1: [10, 10, 10, 10, 10] + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: two-vars-loop-job.yml + tool: loop-inside-loop.cwl + id: loop_nested + doc: "Test a workflow with two nested loops" + output: + o1: [2, 3, 4] + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: two-vars-loop-job.yml + tool: loop-inside-loop-all.cwl + id: loop_nested_all + doc: "Test a workflow with two nested loops, both with outputMethod set to all_iterations" + output: + o1: [[2], [2, 3], [2, 3, 4]] + tags: [ conditional, loop, inline_javascript, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: multi-source-loop.cwl + id: loop_multi_source_input + doc: "Test a loop with two sources, which are selected through a pickValue directive" + output: + o1: [2, 3, 4, 5, 8, 11, 14, 17, 20] + tags: [ conditional, loop, inline_javascript, multiple_input, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: invalid-multi-source-loop-no-requirement.cwl + id: loop_multi_source_input_fail_no_requirement + doc: "Test that a loop with two sources fails without MultipleInputFeatureRequirement" + should_fail: true + tags: [ conditional, loop, inline_javascript, multiple_input, scatter, workflow ] + +- job: single-var-loop-job.yml + tool: default-value-loop.cwl + id: loop_defaultvalue + output: + o1: [8, 11, 14, 17, 20] + doc: "Test a loop whose source has a default value" + tags: [ conditional, loop, inline_javascript, scatter, workflow ] diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl new file mode 100644 index 0000000..decae00 --- /dev/null +++ b/tests/loop/two-vars-loop-2.cwl @@ -0,0 +1,31 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1] diff --git a/tests/loop/two-vars-loop-job.yml b/tests/loop/two-vars-loop-job.yml new file mode 100644 index 0000000..012ed44 --- /dev/null +++ b/tests/loop/two-vars-loop-job.yml @@ -0,0 +1,2 @@ +i1: 1 +i2: 1 \ No newline at end of file diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl new file mode 100644 index 0000000..ccb7e4a --- /dev/null +++ b/tests/loop/two-vars-loop.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: o1 + i2: o2 + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + o2: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2, 'o2': inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1, o2] diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl new file mode 100644 index 0000000..835af1b --- /dev/null +++ b/tests/loop/value-from-loop.cwl @@ -0,0 +1,33 @@ +#!/usr/bin/env cwl-runner +cwlVersion: v1.3.0-dev1 +class: Workflow +requirements: + InlineJavascriptRequirement: {} + StepInputExpressionRequirement: {} +inputs: + i1: int + i2: int +outputs: + o1: + type: int + outputSource: subworkflow/o1 +steps: + subworkflow: + when: $(inputs.i1 < 10) + loop: + i1: + valueFrom: $(inputs.i1 + 1) + outputMethod: last_iteration + run: + class: ExpressionTool + inputs: + i1: int + i2: int + outputs: + o1: int + expression: > + ${return {'o1': inputs.i1 + inputs.i2};} + in: + i1: i1 + i2: i2 + out: [o1]