Merge pull request #1 from common-workflow-language/restructure

Restructure

.dockerignore

@@ -0,0 +1,4 @@
+.venv/
+*.egg-info/
+build/
+dist/

.travis.yml

@@ -0,0 +1,16 @@
+language: python
+matrix:
+  include:
+  - env: TARGET=py3
+  - env: TARGET=pep8
+  - env: TARGET=mypy
+  allow_failures:
+  - env: TARGET=pep8
+  - env: TARGET=mypy
+sudo: required
+services:
+ - docker
+install:
+ - true
+script:
+- docker build . -f .travis/${TARGET}.docker

.travis/mypy.docker

@@ -0,0 +1,6 @@
+FROM kernsuite/base:5
+RUN docker-apt-install python3-pip
+RUN pip3 install mypy
+ADD . /code
+WORKDIR /code
+RUN mypy --ignore-missing-imports cwl_utils

.travis/pep8.docker

@@ -0,0 +1,6 @@
+FROM kernsuite/base:5
+RUN docker-apt-install python3-pip
+RUN pip3 install pycodestyle
+ADD . /code
+WORKDIR /code
+RUN pycodestyle cwl_utils

.travis/py3.docker

@@ -0,0 +1,5 @@
+FROM kernsuite/base:5
+RUN docker-apt-install python3-pip
+ADD . /code
+WORKDIR /code
+RUN pip3 install .

README.md

@@ -7,3 +7,10 @@ source venv3.6/bin/activate
 pip install cwl-utils/docker_cache_pull/requirements.txt
 python docker_cache_pull/docker-extract.py path_to_my_workflow.cwl
 ```
+
+to regenerate install `schema_salad` package and run:
+
+```
+schema-salad-tool --codegen python \
+    https://raw.githubusercontent.com/common-workflow-language/common-workflow-language/master/v1.0/CommonWorkflowLanguage.yml
+```

docker_cache_pull/cite-extract.py → cwl_utils/cite-extract.py

@@ -1,16 +1,19 @@
 #!/usr/bin/env python3
 import sys
-import CommonWorkflowLanguage as cwl
+import cwl_utils.parser_v1_0 as cwl
+
 
 def main():
     top = cwl.load_document(sys.argv[1])
     traverse(top)
 
+
 def extract_software_packages(process: cwl.Process):
     for req in extract_software_reqs(process):
         print(process.id)
         process_software_requirement(req)
 
+
 def extract_software_reqs(process: cwl.Process):
     if process.requirements:
         for req in process.requirements:
@@ -22,22 +25,25 @@ def extract_software_reqs(process: cwl.Process):
                 yield cwl.load_field(req, cwl.SoftwareRequirementLoader,
                                      process.id, process.loadingOptions)
 
-def process_software_requirement(req: cwl.SoftwarePackage):
+
+def process_software_requirement(req: cwl.SoftwareRequirement):
     for package in req.packages:
         print("Package: {}, version: {}, specs: {}".format(
-              package.package, package.version, package.specs))
+            package.package, package.version, package.specs))
 
 
 def traverse(process: cwl.Process):
     extract_software_packages(process)
     if isinstance(process, cwl.Workflow):
         traverse_workflow(process)
 
+
 def get_process_from_step(step: cwl.WorkflowStep):
     if isinstance(step.run, str):
         return cwl.load_document(step.run)
     return step.run
 
+
 def traverse_workflow(workflow: cwl.Workflow):
     for step in workflow.steps:
         extract_software_packages(step)

docker_cache_pull/docker-extract.py → cwl_utils/docker-extract.py

@@ -1,14 +1,15 @@
 #!/usr/bin/env python3
 import sys
 import os
-import CommonWorkflowLanguage as cwl
+import cwl_utils.parser_v1_0 as cwl
 import subprocess
 import argparse
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Tool to save docker images from a cwl workflow \n '
-                                                 'and generate udocker loading commands.')
+    parser = argparse.ArgumentParser(
+        description='Tool to save docker images from a cwl workflow \n '
+                    'and generate udocker loading commands.')
     parser.add_argument('dir', help='Directory in which to save images')
     parser.add_argument('input', help='Input CWL workflow')
     return parser.parse_args()
@@ -36,8 +37,17 @@ def load_docker_image(image_name):
 
 
 def save_docker_image(req, image_name, image_dir):
-    cmd = ['docker', 'save', '-o', os.path.join(image_dir, image_name), req]
-    subprocess.run(" ".join(cmd), shell=True, check=True)
+    cmd_pull = ['docker', 'pull', req]
+    try:
+        subprocess.run(cmd_pull, check=True, stdout=subprocess.PIPE,
+                       stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError as err:
+        if err.output:
+            raise subprocess.SubprocessError(err.output)
+        raise err
+    cmd_save = ['docker', 'save', '-o', os.path.join(image_dir, image_name),
+                req]
+    subprocess.run(cmd_save, check=True)
 
 
 def extract_docker_requirements(process: cwl.Process):
@@ -59,12 +69,6 @@ def extract_docker_reqs(process: cwl.Process):
                                      process.id, process.loadingOptions)
 
 
-def process_docker_requirement(req: cwl.SoftwarePackage):
-    if ':' not in req.dockerPull:
-        req.dockerPull += ':latest'
-    return req.dockerPull
-
-
 def traverse(process: cwl.Process):
     yield from extract_docker_requirements(process)
     if isinstance(process, cwl.Workflow):