Merge branch 'main' into prov_data_input_output

.github/workflows/ci-tests.yml

@@ -173,6 +173,8 @@ jobs:
           - cwl-version: v1.2
             container: docker
             extras: "--fast-parser"
+          - cwl-version: v1.3.0-dev1
+            extras: "--relax-path-checks"
 
     steps:
       - uses: actions/checkout@v4

MANIFEST.in

@@ -7,6 +7,7 @@ recursive-include mypy-stubs *.pyi *.py
 include tests/*
 include tests/cwl-conformance/cwltool-conftest.py
 include tests/loop/*
+include tests/loop-ext/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/tmp4/alpha/*
 include tests/wf/*
@@ -54,6 +55,10 @@ include cwltool/schemas/v1.2/*.yml
 include cwltool/schemas/v1.2/*.md
 include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.yml
 include cwltool/schemas/v1.2/salad/schema_salad/metaschema/*.md
+include cwltool/schemas/v1.3.0-dev1/*.yml
+include cwltool/schemas/v1.3.0-dev1/*.md
+include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.yml
+include cwltool/schemas/v1.3.0-dev1/salad/schema_salad/metaschema/*.md
 include cwltool/extensions.yml
 include cwltool/extensions-v1.1.yml
 include cwltool/extensions-v1.2.yml

Makefile

@@ -160,11 +160,11 @@ diff-cover.html: coverage.xml
 
 ## test                   : run the cwltool test suite
 test: $(PYSOURCES)
-	python3 -m pytest -rs ${PYTEST_EXTRA}
+	python3 -m pytest -rsfE ${PYTEST_EXTRA}
 
 ## testcov                : run the cwltool test suite and collect coverage
 testcov: $(PYSOURCES)
-	python3 -m pytest -rs --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
+	python3 -m pytest -rsfE --cov --cov-config=.coveragerc --cov-report= ${PYTEST_EXTRA}
 
 sloccount.sc: $(PYSOURCES) Makefile
 	sloccount --duplicates --wide --details $^ > $@
@@ -183,7 +183,7 @@ mypy: $(PYSOURCES)
 
 mypyc: $(PYSOURCES)
 	MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
-		 && pytest -rs -vv ${PYTEST_EXTRA}
+		 && pytest -rsfE -vv ${PYTEST_EXTRA}
 
 shellcheck: FORCE
 	shellcheck build-cwltool-docker.sh cwl-docker.sh release-test.sh conformance-test.sh \

conformance-test.sh

@@ -16,7 +16,7 @@ venv() {
 # VERSION=v1.2 GIT_TARGET=main CONTAINER=podman ./conformance_test.sh
 
 # Version of the standard to test against
-# Current options: v1.0, v1.1, v1.2
+# Current options: v1.0, v1.1, v1.2, v1.3.0-dev1
 VERSION=${VERSION:-"v1.2"}
 
 # Which commit of the standard's repo to use
@@ -78,7 +78,7 @@ else
 	pip uninstall -y cwltool
 	pip install -r"${SCRIPT_DIRECTORY}/mypy-requirements.txt"
 	CWLTOOL_USE_MYPYC=1 MYPYPATH="${SCRIPT_DIRECTORY}/mypy-stubs" pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt"
-	pip install 'cwltest>=2.5' pytest-cov pytest-xdist>=3.2.0 psutil
+	pip install 'cwltest>=2.5,!=2.5.20240709060407' pytest-cov pytest-xdist>=3.2.0 psutil
 fi
 
 # Set conformance test filename
@@ -121,7 +121,7 @@ if (( "${#exclusions[*]}" > 0 )); then
 fi
 
 # Build command
-TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rs --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
+TEST_COMMAND="python -m pytest ${CONFORMANCE_TEST} -n logical --dist worksteal -rsfE --junit-xml=${TMP_DIR}/cwltool_conf_${VERSION}_${GIT_TARGET}_${CONTAINER}.xml -o junit_suite_name=cwltool_$(echo "${CWLTOOL_OPTIONS}" | tr "[:blank:]-" _)"
 if [[ -n "${EXCLUDE}" ]] ; then
   TEST_COMMAND="${TEST_COMMAND} --cwl-exclude ${EXCLUDE}"
 fi

cwltool/checker.py

@@ -517,42 +517,33 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
 
 
 def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
-    """Check if a step contains a http://commonwl.org/cwltool#Loop requirement with `all` outputMethod."""
+    """Check if a step contains a `loop` directive with `all` outputMethod."""
     source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
     if source_step is not None:
-        for requirement in source_step.get("requirements", []):
-            if (
-                requirement["class"] == "http://commonwl.org/cwltool#Loop"
-                and requirement.get("outputMethod") == "all"
-            ):
-                return True
+        if source_step.get("loop") is not None and source_step.get("outputMethod") == "all":
+            return True
     return False
 
 
 def loop_checker(steps: Iterator[MutableMapping[str, Any]]) -> None:
     """
-    Check http://commonwl.org/cwltool#Loop requirement compatibility with other directives.
+    Check `loop` compatibility with other directives.
 
-    :raises ValidationException: If there is an incompatible combination between
-            cwltool:loop and 'scatter' or 'when'.
+    :raises ValidationException: If there is an incompatible combination between `loop` and `scatter`.
     """
     exceptions = []
     for step in steps:
-        requirements = {
-            **{h["class"]: h for h in step.get("hints", [])},
-            **{r["class"]: r for r in step.get("requirements", [])},
-        }
-        if "http://commonwl.org/cwltool#Loop" in requirements:
-            if "when" in step:
+        if "loop" in step:
+            if "when" not in step:
                 exceptions.append(
                     SourceLine(step, "id").makeError(
-                        "The `cwltool:Loop` clause is not compatible with the `when` directive."
+                        "The `when` clause is mandatory when the `loop` directive is defined."
                     )
                 )
             if "scatter" in step:
                 exceptions.append(
                     SourceLine(step, "id").makeError(
-                        "The `cwltool:Loop` clause is not compatible with the `scatter` directive."
+                        "The `loop` clause is not compatible with the `scatter` directive."
                     )
                 )
     if exceptions:

cwltool/errors.py

@@ -1,5 +1,17 @@
-class WorkflowException(Exception):
-    pass
+"""Common errors.
+
+WorkflowException and GraphTargetMissingException are aliased to
+equivalent errors from cwl_utils.errors and re-exported by this module
+to avoid breaking the interface for other code.
+
+"""
+
+# flake8: noqa: F401
+
+from cwl_utils.errors import WorkflowException as WorkflowException
+
+
+from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException
 
 
 class UnsupportedRequirement(WorkflowException):
@@ -8,7 +20,3 @@ class UnsupportedRequirement(WorkflowException):
 
 class ArgumentException(Exception):
     """Mismatched command line arguments provided."""
-
-
-class GraphTargetMissingException(WorkflowException):
-    """When a ``$graph`` is encountered and there is no target and no ``main``/``#main``."""

cwltool/extensions-v1.3.yml

@@ -0,0 +1,121 @@
+$base: http://commonwl.org/cwltool#
+$namespaces:
+  cwl: "https://w3id.org/cwl/cwl#"
+$graph:
+- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
+
+- name: Secrets
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  fields:
+    class:
+      type: string
+      doc: "Always 'Secrets'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    secrets:
+      type: string[]
+      doc: |
+        List one or more input parameters that are sensitive (such as passwords)
+        which will be deliberately obscured from logging.
+      jsonldPredicate:
+        "_type": "@id"
+        refScope: 0
+
+
+- name: ProcessGenerator
+  type: record
+  inVocab: true
+  extends: cwl:Process
+  documentRoot: true
+  fields:
+    - name: class
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+      type: string
+    - name: run
+      type: [string, cwl:Process]
+      jsonldPredicate:
+        _id: "cwl:run"
+        _type: "@id"
+        subscope: run
+      doc: |
+        Specifies the process to run.
+
+- name: MPIRequirement
+  type: record
+  inVocab: false
+  extends: cwl:ProcessRequirement
+  doc: |
+    Indicates that a process requires an MPI runtime.
+  fields:
+    - name: class
+      type: string
+      doc: "Always 'MPIRequirement'"
+      jsonldPredicate:
+        "_id": "@type"
+        "_type": "@vocab"
+    - name: processes
+      type: [int, cwl:Expression]
+      doc: |
+        The number of MPI processes to start. If you give a string,
+        this will be evaluated as a CWL Expression and it must
+        evaluate to an integer.
+
+- name: CUDARequirement
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Require support for NVIDA CUDA (GPU hardware acceleration).
+  fields:
+    class:
+      type: string
+      doc: 'cwltool:CUDARequirement'
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    cudaVersionMin:
+      type: string
+      doc: |
+        Minimum CUDA version to run the software, in X.Y format.  This
+        corresponds to a CUDA SDK release.  When running directly on
+        the host (not in a container) the host must have a compatible
+        CUDA SDK (matching the exact version, or, starting with CUDA
+        11.3, matching major version).  When run in a container, the
+        container image should provide the CUDA runtime, and the host
+        driver is injected into the container.  In this case, because
+        CUDA drivers are backwards compatible, it is possible to
+        use an older SDK with a newer driver across major versions.
+
+        See https://docs.nvidia.com/deploy/cuda-compatibility/ for
+        details.
+    cudaComputeCapability:
+      type:
+        - 'string'
+        - 'string[]'
+      doc: |
+        CUDA hardware capability required to run the software, in X.Y
+        format.
+
+        * If this is a single value, it defines only the minimum
+          compute capability.  GPUs with higher capability are also
+          accepted.
+
+        * If it is an array value, then only select GPUs with compute
+          capabilities that explicitly appear in the array.
+    cudaDeviceCountMin:
+      type: ['null', int, cwl:Expression]
+      default: 1
+      doc: |
+        Minimum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMax`.  If neither are specified,
+        default 1.
+    cudaDeviceCountMax:
+      type: ['null', int, cwl:Expression]
+      doc: |
+        Maximum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMin`.

cwltool/job.py

@@ -917,7 +917,7 @@ def docker_monitor(
             self.name,
             int((max_mem_percent / 100 * max_mem) / (2**20)),
         )
-        if cleanup_cidfile:
+        if cleanup_cidfile and os.path.exists(cidfile):
             os.remove(cidfile)
 
 

cwltool/process.py

@@ -125,6 +125,7 @@ def filter(self, record: logging.LogRecord) -> bool:
 ]
 
 cwl_files = (
+    "Base.yml",
     "Workflow.yml",
     "CommandLineTool.yml",
     "CommonWorkflowLanguage.yml",
@@ -1046,10 +1047,6 @@ def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict:
             sl = SourceLine(hints, i, ValidationException, debug)
             with sl:
                 classname = cast(str, r["class"])
-                if classname == "http://commonwl.org/cwltool#Loop":
-                    raise ValidationException(
-                        "http://commonwl.org/cwltool#Loop is valid only under requirements."
-                    )
                 avroname = classname
                 if classname in self.doc_loader.vocab:
                     avroname = avro_type_name(self.doc_loader.vocab[classname])