Ci/ci and pre-commit updates

.autorc

@@ -0,0 +1,7 @@
+{
+    "onlyPublishWithReleaseLabel": true,
+    "baseBranch": "master",
+    "author": "Nipype Bot <nipype@mit.edu>",
+    "noVersionPrefix": true,
+    "plugins": ["git-tag"]
+}

.codespell-ignorewords

@@ -0,0 +1 @@
+nd

.github/workflows/release.yml

@@ -0,0 +1,44 @@
+name: Auto-release on PR merge
+
+on:
+  # ATM, this is the closest trigger to a PR merging
+  push:
+    branches:
+      - master
+
+env:
+  # Pin to v10.28.0, which (as of 2021-05-25) is the latest version with assets
+  AUTO_VERSION: v10.29.3
+
+jobs:
+  auto-release:
+    runs-on: ubuntu-latest
+    if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')"
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Prepare repository
+        # Fetch full git history and tags
+        run: git fetch --unshallow --tags
+
+      - name: Unset header
+        # checkout@v2 adds a header that makes branch protection report errors
+        # because the Github action bot is not a collaborator on the repo
+        run: git config --local --unset http.https://github.com/.extraheader
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+
+      - name: Download auto
+        run: |
+          auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/tags/$AUTO_VERSION | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')"
+          wget -O- "$auto_download_url" | gunzip > ~/auto
+          chmod a+x ~/auto
+
+      - name: Create release
+        run: |
+          ~/auto shipit -vv
+        env:
+          GH_TOKEN: ${{ secrets.AUTO_USER_TOKEN }}

.github/workflows/teststyle.yml

@@ -4,19 +4,14 @@ on: [push, pull_request]
 
 jobs:
   build:
-    strategy:
-      matrix:
-        python-version: [3.7, 3.8, 3.9]
-      fail-fast: false
     runs-on: ubuntu-latest
 
-
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }} on ubuntu-latest
+    - name: Set up Python 3.x on ubuntu-latest
       uses: actions/setup-python@v2
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: '3.x'
     - name: Update build tools
       run: python -m pip install --upgrade pip setuptools
 
@@ -25,5 +20,5 @@ jobs:
 
     - name: Check Style
       run: |
-        pip install black==21.4b2 codecov
+        pip install black==21.8b0 codecov
         black --check pydra setup.py

.pre-commit-config.yaml

@@ -12,3 +12,12 @@ repos:
     rev: 21.8b0
     hooks:
     -   id: black
+-   repo: https://github.com/codespell-project/codespell
+    rev: v2.0.0
+    hooks:
+    -   id: codespell
+        exclude: ^(pydra/_version\.py|versioneer\.py)$
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+      - id: flake8

docs/changes.rst

@@ -46,7 +46,7 @@ Release Notes
 * adding ``wf._connection`` to ``checksum``
 * allowing for updates of ``wf._connections``
 * editing output, so it works with ``numpy.arrays``
-* removing ``to_job`` and pickling task instead (workers read the tasks and set the proper input, so the multile copies of the input are not kept in the memory)
+* removing ``to_job`` and pickling task instead (workers read the tasks and set the proper input, so the multiple copies of the input are not kept in the memory)
 * adding standalone function ``load_and_run`` that can load and run a task from a pickle file
 * removing ``create_pyscript`` and simplifying the slurm worker
 * improving error reports in errors flies

docs/components.rst

@@ -148,10 +148,10 @@ Workflows
     # to the workflow input
     wf.add(mult(name="mlt",
                    x=wf.lzin.x, y=wf.lzin.y))
-    # adding anoter task and connecting
+    # adding another task and connecting
     # task's input to the "mult" task's output
     wf.add(add2(name="add", x=wf.mlt.lzout.out))
-    # setting worflow output
+    # setting workflow output
     wf.set_output([("out", wf.add.lzout.out)])
 
 

docs/index.rst

@@ -60,7 +60,7 @@ The combination of several key features makes Pydra a customizable and powerful
   (via Docker or Singularity) enabling greater consistency for reproducibility.
 
 - Auditing and provenance tracking: Pydra provides a simple JSON-LD-based message passing mechanism
-  to capture the dataflow execution activties as a provenance graph. These messages track inputs
+  to capture the dataflow execution activities as a provenance graph. These messages track inputs
   and outputs of each task in a dataflow, and the resources consumed by the task.
 
 .. _Nipype: https://nipype.readthedocs.io/en/latest/

docs/output_spec.rst

@@ -85,6 +85,6 @@ The metadata dictionary for `output_spec` can include:
 `callable` (`function`):
    If provided the output file name (or list of file names) is created using the function.
    The function can take `field` (the specific output field will be passed to the function),
-   `output_dir` (task `output_dir` wil be used), `stdout`, `stderr` (`stdout` and `stderr` of
+   `output_dir` (task `output_dir` will be used), `stdout`, `stderr` (`stdout` and `stderr` of
    the task will be sent) `inputs` (entire `inputs` will be passed) or any input field name
    (a specific input field will be sent).

pydra/engine/audit.py

@@ -18,7 +18,8 @@ def __init__(self, audit_flags, messengers, messenger_args, develop=None):
         ----------
         audit_flags : :class:`AuditFlag`
             Base configuration of auditing.
-        messengers : :obj:`pydra.util.messenger.Messenger` or list of :class:`pydra.util.messenger.Messenger`, optional
+        messengers : :obj:`pydra.util.messenger.Messenger`
+        or list of :class:`pydra.util.messenger.Messenger`, optional
             Specify types of messenger used by Audit to send a message.
             Could be `PrintMessenger`, `FileMessenger`, or `RemoteRESTMessenger`.
         messenger_args : :obj:`dict`, optional

pydra/engine/boutiques.py

@@ -89,7 +89,7 @@ def __init__(
 
     def _download_spec(self, zenodo_id):
         """
-        usind boutiques Searcher to find url of zenodo file for a specific id,
+        using boutiques Searcher to find url of zenodo file for a specific id,
         and download the file to self.cache_dir
         """
         from boutiques.searcher import Searcher

pydra/engine/core.py

@@ -3,7 +3,8 @@
 import attr
 import json
 import logging
-import os, sys
+import os
+import sys
 from pathlib import Path
 import typing as ty
 from copy import deepcopy
@@ -150,7 +151,8 @@ def __init__(
 
         if inputs:
             if isinstance(inputs, dict):
-                # selecting items that are in input_names (ignoring fields that are not in input_spec)
+                # selecting items that are in input_names
+                # (ignoring fields that are not in input_spec)
                 inputs = {k: v for k, v in inputs.items() if k in self.input_names}
             # TODO: this needs to finished and tested after #305
             elif Path(inputs).is_file():
@@ -240,7 +242,7 @@ def errored(self):
     def checksum(self):
         """Calculates the unique checksum of the task.
         Used to create specific directory name for task that are run;
-        and to create nodes checksums needed for graph checkums
+        and to create nodes checksums needed for graph checksums
         (before the tasks have inputs etc.)
         """
         input_hash = self.inputs.hash
@@ -286,8 +288,6 @@ def checksum_states(self, state_index=None):
                 if val:
                     self.inputs.files_hash[key].update(val)
             if is_workflow(self):
-                con_hash = hash_function(self._connections)
-                hash_list = [input_hash, con_hash]
                 checksum_ind = create_checksum(
                     self.__class__.__name__, self._checksum_wf(input_hash)
                 )
@@ -464,7 +464,7 @@ def _run(self, rerun=False, **kwargs):
                 self.audit.monitor()
                 self._run_task()
                 result.output = self._collect_outputs(output_dir=odir)
-            except Exception as e:
+            except Exception:
                 etype, eval, etr = sys.exc_info()
                 traceback = format_exception(etype, eval, etr)
                 record_error(self.output_dir, error=traceback)
@@ -497,7 +497,7 @@ def _collect_outputs(self, output_dir):
 
     def split(self, splitter, overwrite=False, cont_dim=None, **kwargs):
         """
-        Run this task parametrically over lists of splitted inputs.
+        Run this task parametrically over lists of split inputs.
 
         Parameters
         ----------
@@ -585,7 +585,7 @@ def _extract_input_el(self, inputs, inp_nm, ind):
     def get_input_el(self, ind):
         """Collect all inputs required to run the node (for specific state element)."""
         if ind is not None:
-            # TODO: doesnt work properly for more cmplicated wf (check if still an issue)
+            # TODO: doesn't work properly for more cmplicated wf (check if still an issue)
             state_dict = self.state.states_val[ind]
             input_ind = self.state.inputs_ind[ind]
             inputs_dict = {}
@@ -864,7 +864,7 @@ def graph_sorted(self):
     def checksum(self):
         """Calculates the unique checksum of the task.
         Used to create specific directory name for task that are run;
-        and to create nodes checksums needed for graph checkums
+        and to create nodes checksums needed for graph checksums
         (before the tasks have inputs etc.)
         """
         # if checksum is called before run the _graph_checksums is not ready
@@ -1035,7 +1035,7 @@ async def _run(self, submitter=None, rerun=False, **kwargs):
                 self.audit.monitor()
                 await self._run_task(submitter, rerun=rerun)
                 result.output = self._collect_outputs()
-            except Exception as e:
+            except Exception:
                 etype, eval, etr = sys.exc_info()
                 traceback = format_exception(etype, eval, etr)
                 record_error(self.output_dir, error=traceback)
@@ -1113,7 +1113,6 @@ def set_output(self, connections):
         logger.info("Added %s to %s", self.output_spec, self)
 
     def _collect_outputs(self):
-        error = False
         output_klass = make_klass(self.output_spec)
         output = output_klass(**{f.name: None for f in attr.fields(output_klass)})
         # collecting outputs from tasks
@@ -1134,7 +1133,8 @@ def _collect_outputs(self):
                 else:
                     raise ValueError(
                         f"Task {val.name} raised an error, "
-                        f"full crash report is here: {getattr(self, val.name).output_dir / '_error.pklz'}"
+                        f"full crash report is here: "
+                        f"{getattr(self, val.name).output_dir / '_error.pklz'}"
                     )
         return attr.evolve(output, **output_wf)
 

pydra/engine/graph.py

@@ -242,7 +242,7 @@ def remove_nodes(self, nodes, check_ready=True):
             if nd not in self.nodes:
                 raise Exception(f"{nd} is not present in the graph")
             if self.predecessors[nd.name] and check_ready:
-                raise Exception("this node shoudn't be run, has to wait")
+                raise Exception("this node shouldn't be run, has to wait")
             self.nodes.remove(nd)
             # adding the node to self._node_wip as for
             self._node_wip.append(nd)
@@ -398,13 +398,19 @@ def create_dotfile_detailed(self, outdir, name="graph_det"):
                 for el in nd_det["outputs"][1:]:
                     wf_inputs_str += f" | <{el}> {el}"
                 wf_inputs_str += "}"
-                dotstr += f'struct_{nd_nm} [color=red, label="{{WORKFLOW INPUT: | {wf_inputs_str}}}"];\n'
+                dotstr += (
+                    f"struct_{nd_nm} "
+                    f'[color=red, label="{{WORKFLOW INPUT: | {wf_inputs_str}}}"];\n'
+                )
                 # wf outputs
                 wf_outputs_str = f'{{<{nd_det["inputs"][0]}> {nd_det["inputs"][0]}'
                 for el in nd_det["inputs"][1:]:
                     wf_outputs_str += f" | <{el}> {el}"
                 wf_outputs_str += "}"
-                dotstr += f'struct_{nd_nm}_out [color=red, label="{{WORKFLOW OUTPUT: | {wf_outputs_str}}}"];\n'
+                dotstr += (
+                    f"struct_{nd_nm}_out "
+                    f'[color=red, label="{{WORKFLOW OUTPUT: | {wf_outputs_str}}}"];\n'
+                )
                 # connections to the wf outputs
                 for con in nd_det["connections"]:
                     dotstr += (
@@ -419,7 +425,10 @@ def create_dotfile_detailed(self, outdir, name="graph_det"):
                 for out in nd_det["outputs"]:
                     outputs_str += f" | <{out}> {out}"
                 outputs_str += "}"
-                dotstr += f'struct_{nd_nm} [shape=record, label="{inputs_str} | {nd_nm} | {outputs_str}"];\n'
+                dotstr += (
+                    f"struct_{nd_nm} "
+                    f'[shape=record, label="{inputs_str} | {nd_nm} | {outputs_str}"];\n'
+                )
                 # connections between elements
                 for con in nd_det["connections"]:
                     dotstr += f"struct_{con[1]}:{con[2]} -> struct_{nd_nm}:{con[0]};\n"
@@ -545,7 +554,7 @@ def export_graph(self, dotfile, ext="png"):
             "xdot_json",
         ]
         if ext not in available_ext:
-            raise Exception(f"unvalid extension - {ext}, chose from {available_ext}")
+            raise Exception(f"invalid extension - {ext}, chose from {available_ext}")
 
         dot_check = sp.run(["which", "dot"], stdout=sp.PIPE, stderr=sp.PIPE)
         if not dot_check.stdout:

pydra/engine/helpers.py

@@ -354,7 +354,8 @@ def custom_validator(instance, attribute, value):
                 tp = {"key": tp_k, "val": tp_v}
         else:
             warnings.warn(
-                f"no type check for {attribute.name} field, no type check implemented for value {value} and type {tp_attr}"
+                f"no type check for {attribute.name} field, "
+                f"no type check implemented for value {value} and type {tp_attr}"
             )
             check_type = False
 
@@ -401,7 +402,8 @@ def _type_validator(instance, attribute, value, tp, cont_type):
         )(instance, attribute, value)
     else:
         raise Exception(
-            f"container type of {attribute.name} should be None, list, dict or ty.Union, and not {cont_type}"
+            f"container type of {attribute.name} should be None, list, dict or ty.Union, "
+            f"and not {cont_type}"
         )
 
 
@@ -450,7 +452,8 @@ def _single_type_update(tp, name, simplify=False):
             return None
     else:
         warnings.warn(
-            f"no type check for {name} field, type check not implemented for type - {tp}, consider using simplify=True"
+            f"no type check for {name} field, type check not implemented for type - {tp}, "
+            f"consider using simplify=True"
         )
         return None
 
@@ -727,7 +730,7 @@ def output_from_inputfields(output_spec, input_spec):
                 field_name = fld.metadata["output_field_name"]
             else:
                 field_name = fld.name
-            # not adding if the field already in teh output_spec
+            # not adding if the field already in the output_spec
             if field_name not in current_output_spec_names:
                 # TODO: should probably remove some of the keys
                 new_fields.append(
@@ -774,11 +777,10 @@ def load_and_run(
     """
     try:
         task = load_task(task_pkl=task_pkl, ind=ind)
-    except Exception as excinfo:
+    except Exception:
         if task_pkl.parent.exists():
             etype, eval, etr = sys.exc_info()
             traceback = format_exception(etype, eval, etr)
-            errorfile = record_error(task_pkl.parent, error=traceback)
             result = Result(output=None, runtime=None, errored=True)
             save(task_pkl.parent, result=result)
         raise
@@ -830,7 +832,7 @@ def position_sort(args):
     """
     Sort objects by position, following Python indexing conventions.
 
-    Ordering is postive positions, lowest to highest, followed by unspecified
+    Ordering is positive positions, lowest to highest, followed by unspecified
     positions (``None``) and negative positions, lowest to highest.
 
     >>> position_sort([(None, "d"), (-3, "e"), (2, "b"), (-2, "f"), (5, "c"), (1, "a")])
@@ -871,8 +873,8 @@ def argstr_formatting(argstr, inputs, value_updates=None):
     if value_updates:
         inputs_dict.update(value_updates)
     # getting all fields that should be formatted, i.e. {field_name}, ...
-    inp_fields = re.findall("{\w+}", argstr)
-    inp_fields_float = re.findall("{\w+:[0-9.]+f}", argstr)
+    inp_fields = re.findall(r"{\w+}", argstr)
+    inp_fields_float = re.findall(r"{\w+:[0-9.]+f}", argstr)
     inp_fields += [re.sub(":[0-9.]+f", "", el) for el in inp_fields_float]
     val_dict = {}
     for fld in inp_fields: