Merge branch 'main' into check_singularity_image

Makefile

@@ -24,7 +24,7 @@ MODULE=cwl_utils
 
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
-PYSOURCES=$(filter-out cwl_utils/parser_v%,$(wildcard ${MODULE}/**.py tests/*.py)) setup.py
+PYSOURCES=$(filter-out cwl_utils/parser/cwl_v%,$(wildcard ${MODULE}/**.py tests/*.py)) setup.py
 DEVPKGS=diff_cover black pylint coverage pep257 pydocstyle flake8 mypy\
 	isort wheel autoflake
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \

README.md

@@ -56,34 +56,33 @@ from pathlib import Path
 from ruamel import yaml
 import sys
 
+from cwl_utils.parser import cwl_version, load_document, save
+
 # File Input - This is the only thing you will need to adjust or take in as an input to your function:
 cwl_file = Path("/path/to/wf.cwl")
 
 # Read in the cwl file from a yaml
 with open(cwl_file, "r") as cwl_h:
- yaml_obj = yaml.main.round_trip_load(cwl_h, preserve_quotes=True)
+    yaml_obj = yaml.main.round_trip_load(cwl_h, preserve_quotes=True)
 
 # Check CWLVersion
-if 'cwlVersion' not in list(yaml_obj.keys()):
-    print("Error - could not get the cwlVersion")
-    sys.exit(1)
-
-# Import parser based on CWL Version    
-if yaml_obj['cwlVersion'] == 'v1.0':
-    from cwl_utils import parser_v1_0 as parser
-elif yaml_obj['cwlVersion'] == 'v1.1':
-    from cwl_utils import parser_v1_1 as parser
-elif yaml_obj['cwlVersion'] == 'v1.2':
-    from cwl_utils import parser_v1_2 as parser
-else:
-    print("Version error. Did not recognise {} as a CWL version".format(yaml_obj["cwlVersion"]))
+try:
+    ver = cwl_version(yaml_obj)
+    if ver is None:
+        print("Error - could not get the cwlVersion")
+        sys.exit(1)
+except ValidationException as e:
+    print("Error - yaml_obj is not a mapping")
     sys.exit(1)
 
 # Import CWL Object
-cwl_obj = parser.load_document_by_yaml(yaml_obj, cwl_file.as_uri())
+cwl_obj = load_document(yaml_obj, cwl_file.as_uri())
 
 # View CWL Object
 print("List of object attributes:\n{}".format("\n".join(map(str, dir(cwl_obj)))))
+
+# Export CWL Object into a built-in typed object
+saved_obj = save(cwl_obj)
 ```
 
 ## Development
@@ -92,13 +91,13 @@ print("List of object attributes:\n{}".format("\n".join(map(str, dir(cwl_obj))))
 
 To regenerate install the `schema_salad` package and run:
 
-`cwl_utils/parser_v1_0.py` was created via
+`cwl_utils/parser/cwl_v1_0.py` was created via
 `schema-salad-tool --codegen python https://github.com/common-workflow-language/common-workflow-language/raw/main/v1.0/CommonWorkflowLanguage.yml`
 
-`cwl_utils/parser_v1_1.py` was created via
+`cwl_utils/parser/cwl_v1_1.py` was created via
 `schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.1/raw/main/CommonWorkflowLanguage.yml`
 
-`cwl_utils/parser_v1_2.py` was created via
+`cwl_utils/parser/cwl_v1_2.py` was created via
 `schema-salad-tool --codegen python https://github.com/common-workflow-language/cwl-v1.2/raw/main/CommonWorkflowLanguage.yml`
 
 

create_cwl_from_objects.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 # SPDX-License-Identifier: Apache-2.0
-import cwl_utils.parser_v1_2 as cwl
+import cwl_utils.parser.parser_v1_2 as cwl
 
 from ruamel import yaml
 

cwl_utils/cite_extract.py

@@ -3,7 +3,7 @@
 import sys
 from typing import Iterator, Union, cast
 
-import cwl_utils.parser_v1_0 as cwl
+import cwl_utils.parser.cwl_v1_0 as cwl
 
 ProcessType = Union[cwl.Workflow, cwl.CommandLineTool, cwl.ExpressionTool]
 

cwl_utils/cwl_expression_refactor.py

@@ -23,9 +23,12 @@
     cwl_v1_0_expression_refactor,
     cwl_v1_1_expression_refactor,
     cwl_v1_2_expression_refactor,
-    parser_v1_0,
-    parser_v1_1,
-    parser_v1_2,
+)
+
+from cwl_utils.parser import (
+    cwl_v1_0,
+    cwl_v1_1,
+    cwl_v1_2,
 )
 
 save_type = Union[Dict[str, str], List[Union[Dict[str, str], List[Any], None]], None]
@@ -92,19 +95,19 @@ def run(args: argparse.Namespace) -> int:
         version = result["cwlVersion"]
         uri = Path(document).resolve().as_uri()
         if version == "v1.0":
-            top = parser_v1_0.load_document_by_yaml(result, uri)
+            top = cwl_v1_0.load_document_by_yaml(result, uri)
             traverse: Callable[
                 [Any, bool, bool, bool, bool], Tuple[Any, bool]
             ] = cwl_v1_0_expression_refactor.traverse
-            save: saveCWL = parser_v1_0.save
+            save: saveCWL = cwl_v1_0.save
         elif version == "v1.1":
-            top = parser_v1_1.load_document_by_yaml(result, uri)
+            top = cwl_v1_1.load_document_by_yaml(result, uri)
             traverse = cwl_v1_1_expression_refactor.traverse
-            save = parser_v1_1.save
+            save = cwl_v1_1.save
         elif version == "v1.2":
-            top = parser_v1_2.load_document_by_yaml(result, uri)
+            top = cwl_v1_2.load_document_by_yaml(result, uri)
             traverse = cwl_v1_2_expression_refactor.traverse
-            save = parser_v1_2.save
+            save = cwl_v1_2.save
         else:
             _logger.error(
                 "Sorry, %s is not a supported CWL version by this tool.", version

cwl_utils/cwl_normalizer.py

@@ -34,7 +34,7 @@
 _cwltoollogger.setLevel(100)
 
 from cwl_utils import cwl_v1_2_expression_refactor
-from cwl_utils.parser_v1_2 import load_document_by_yaml, save
+from cwl_utils.parser.cwl_v1_2 import load_document_by_yaml, save
 
 
 def parse_args(args: List[str]) -> argparse.Namespace:

cwl_utils/cwl_v1_0_expression_refactor.py

@@ -27,7 +27,7 @@
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 
-import cwl_utils.parser_v1_0 as cwl
+import cwl_utils.parser.cwl_v1_0 as cwl
 
 
 def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool:

cwl_utils/cwl_v1_1_expression_refactor.py

@@ -27,7 +27,7 @@
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 
-import cwl_utils.parser_v1_1 as cwl
+import cwl_utils.parser.cwl_v1_1 as cwl
 
 
 def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool:

cwl_utils/cwl_v1_2_expression_refactor.py

@@ -27,7 +27,7 @@
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import json_dumps
 
-import cwl_utils.parser_v1_2 as cwl
+import cwl_utils.parser.cwl_v1_2 as cwl
 
 
 def expand_stream_shortcuts(process: cwl.CommandLineTool) -> cwl.CommandLineTool:

cwl_utils/docker_extract.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import Iterator, Union, cast
 
-import cwl_utils.parser_v1_0 as cwl
+import cwl_utils.parser.cwl_v1_0 as cwl
 from cwl_utils.image_puller import (
     DockerImagePuller,
     ImagePuller,

cwl_utils/parser/__init__.py

@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: Apache-2.0
+
+from schema_salad.exceptions import ValidationException
+from schema_salad.utils import yaml_no_ts
+from . import cwl_v1_0 as cwl_v1_0
+from . import cwl_v1_1 as cwl_v1_1
+from . import cwl_v1_2 as cwl_v1_2
+
+import os
+from typing import cast, Any, MutableMapping, MutableSequence, Union, Optional
+
+LoadingOptions = Union[
+    cwl_v1_0.LoadingOptions, cwl_v1_1.LoadingOptions, cwl_v1_2.LoadingOptions
+]
+Savable = Union[cwl_v1_0.Savable, cwl_v1_1.Savable, cwl_v1_2.Savable]
+
+
+def cwl_version(yaml: Any) -> Any:
+    """Return the cwlVersion of a YAML object.
+
+    Args:
+        yaml: A YAML object
+
+    Returns:
+        Any: The value of `cwlVersion`. Its type is Optional[str] when a given YAML object is a valid CWL object.
+
+    Raises:
+        ValidationException: If `yaml` is not a MutableMapping.
+    """
+    if not isinstance(yaml, MutableMapping):
+        raise ValidationException("MutableMapping is required")
+    if "cwlVersion" not in list(yaml.keys()):
+        return None
+    return yaml["cwlVersion"]
+
+
+def load_document(
+    doc: Any,
+    baseuri: Optional[str] = None,
+    loadingOptions: Optional[LoadingOptions] = None,
+) -> Any:
+    """Load a CWL object from a serialized YAML string or a YAML object."""
+    if baseuri is None:
+        baseuri = cwl_v1_0.file_uri(os.getcwd()) + "/"
+    if isinstance(doc, str):
+        return load_document_by_string(doc, baseuri, loadingOptions)
+    return load_document_by_yaml(doc, baseuri, loadingOptions)
+
+
+def load_document_by_string(
+    string: str, uri: str, loadingOptions: Optional[LoadingOptions] = None
+) -> Any:
+    """Load a CWL object from a serialized YAML string."""
+    yaml = yaml_no_ts()
+    result = yaml.load(string)
+    return load_document_by_yaml(result, uri, loadingOptions)
+
+
+def load_document_by_yaml(
+    yaml: Any, uri: str, loadingOptions: Optional[LoadingOptions] = None
+) -> Any:
+    """Load a CWL object from a YAML object."""
+    version = cwl_version(yaml)
+    if version == "v1.0":
+        result = cwl_v1_0.load_document_by_yaml(
+            yaml, uri, cast(Optional[cwl_v1_0.LoadingOptions], loadingOptions)
+        )
+    elif version == "v1.1":
+        result = cwl_v1_1.load_document_by_yaml(
+            yaml, uri, cast(Optional[cwl_v1_1.LoadingOptions], loadingOptions)
+        )
+    elif version == "v1.2":
+        result = cwl_v1_2.load_document_by_yaml(
+            yaml, uri, cast(Optional[cwl_v1_2.LoadingOptions], loadingOptions)
+        )
+    elif version is None:
+        raise ValidationException("could not get the cwlVersion")
+    else:
+        raise ValidationException(
+            "Version error. Did not recognise {} as a CWL version".format(version)
+        )
+
+    if isinstance(result, MutableSequence):
+        lst = []
+        for r in result:
+            if "cwlVersion" in r.attrs:
+                r.cwlVersion = version
+            lst.append(r)
+        return lst
+    return result
+
+
+def save(
+    val: Optional[Union[Savable, MutableSequence[Savable]]],
+    top: bool = True,
+    base_url: str = "",
+    relative_uris: bool = True,
+) -> Any:
+    """Convert a given CWL object into a build-in typed object."""
+    if (
+        isinstance(val, cwl_v1_0.Savable)
+        or isinstance(val, cwl_v1_1.Savable)
+        or isinstance(val, cwl_v1_2.Savable)
+    ):
+        return val.save(top=top, base_url=base_url, relative_uris=relative_uris)
+    if isinstance(val, MutableSequence):
+        lst = [
+            save(v, top=top, base_url=base_url, relative_uris=relative_uris)
+            for v in val
+        ]
+        if top and all((is_process(v) for v in val)):
+            vers = [
+                e.get("cwlVersion") for i, e in enumerate(lst) if is_process(val[i])
+            ]
+            latest = max(
+                (v for v in vers if v is not None), key=cast(Any, version_split)
+            )
+            return {"cwlVersion": latest, "$graph": lst}
+        return lst
+    if isinstance(val, MutableMapping):
+        newdict = {}
+        for key in val:
+            newdict[key] = save(
+                val[key], top=False, base_url=base_url, relative_uris=relative_uris
+            )
+        return newdict
+    return val
+
+
+def is_process(v: Any) -> bool:
+    return (
+        isinstance(v, cwl_v1_0.Process)
+        or isinstance(v, cwl_v1_1.Process)
+        or isinstance(v, cwl_v1_2.Process)
+    )
+
+
+def version_split(version: str) -> MutableSequence[int]:
+    return [int(v) for v in version[1:].split(".")]

setup.py

@@ -13,8 +13,12 @@
     license="Apache 2.0",
     author="Common workflow language working group",
     author_email="common-workflow-language@googlegroups.com",
-    packages=["cwl_utils", "cwl_utils.tests", "cwl_utils.testdata"],
-    package_dir={"cwl_utils.tests": "tests", "cwl_utils.testdata": "testdata"},
+    packages=["cwl_utils", "cwl_utils.parser", "cwl_utils.tests", "cwl_utils.testdata"],
+    package_dir={
+        "cwl_utils.parser": "cwl_utils/parser",
+        "cwl_utils.tests": "tests",
+        "cwl_utils.testdata": "testdata",
+    },
     include_package_data=True,
     python_requires=">=3.6",
     install_requires=[

tests/test_cite_extract.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-import cwl_utils.parser_v1_0 as parser
+import cwl_utils.parser.cwl_v1_0 as parser
 from cwl_utils.cite_extract import traverse_workflow
 
 HERE = Path(__file__).resolve().parent

tests/test_docker_extract.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 from tempfile import TemporaryDirectory
 
-import cwl_utils.parser_v1_0 as parser
+import cwl_utils.parser.cwl_v1_0 as parser
 from cwl_utils.docker_extract import traverse
 from cwl_utils.image_puller import DockerImagePuller, SingularityImagePuller
 

tests/test_etools_to_clt.py

@@ -11,9 +11,9 @@
 from cwltool.errors import WorkflowException
 from pytest import raises
 
-import cwl_utils.parser_v1_0 as parser
-import cwl_utils.parser_v1_1 as parser1
-import cwl_utils.parser_v1_2 as parser2
+import cwl_utils.parser.cwl_v1_0 as parser
+import cwl_utils.parser.cwl_v1_1 as parser1
+import cwl_utils.parser.cwl_v1_2 as parser2
 from cwl_utils.cwl_v1_0_expression_refactor import traverse as traverse0
 from cwl_utils.cwl_v1_1_expression_refactor import traverse as traverse1
 from cwl_utils.cwl_v1_2_expression_refactor import traverse as traverse2